hacklab/OASIS_MOBILE
1
0
Fork 0
Code Issues Pull requests Projects Releases 1 Packages Wiki Activity Actions

upstream: integrar Oasis 1.3.1

Browse source 
Base del merge a tres bandas: git 0.9.5 de epsylon, esta rama y el src de la
APK oficial 1.3.1 (la variante movil: OASIS_MOBILE, regex sin \p{}, audio y
video de Android). Corre en el mismo nodejs-mobile 18.20.4 que ya usabamos.

Se conserva de la rama: Karvan, identidades y multicuenta, topbar y barra
inferior, borrado de EXIF sin sharp, filtro hot del foro, RTC sin STUN y los
modulos apagados en movil (wallet, banking, torrents).

Se toma de upstream lo que ya hace de serie: respuesta citada en chats, ssb-lan
en movil, configuracion en ~/.ssb/oasis (sobrevive a reinstalar el backend).
El asistente de bienvenida deja elegir workflow tambien en movil.
This commit is contained in:
oasis-bot 2026-10-18 02:52:17 +02:00
parent 8d838dd8b1
commit c332afd6ea
265 changed files with 80970 additions and 29167 deletions
Download patch file Download diff file Expand all files Collapse all files

170
src/models/activity_model.js
View file

@ -1,8 +1,10 @@
const pull = require('../server/node_modules/pull-stream');
const longText = require('../backend/long_text');
const ssbRef = require('../server/node_modules/ssb-ref');
const { getConfig } = require('../configs/config-manager.js');
const { buildVoteTally } = require('../backend/vote_tally');
const { buildValidatedTombstoneSet } = require('./tombstone_validator');
const { versionGroups, versionEntry } = require('../backend/dedupe');
const logLimit = getConfig().ssbLogStream?.limit || 1000;
const safeFeedId = (v) => {
@ -57,6 +59,8 @@ function inferType(c = {}) {
}
const HIDDEN_ENVELOPE_TYPES = new Set([
'textChunk',
'aiExchange',
'tribe-keys-distrib',
'tribe-keys',
'tribe-invite-msg',
@ -67,7 +71,6 @@ const HIDDEN_ENVELOPE_TYPES = new Set([
'larpJoinHouse',
'larpLeaveLarp',
'larpTestAttempt',
'larpHousePost',
'larpHouseInvite',
'larpHouseInviteRedeem',
'larpHouseTribeAnchor',
@ -75,19 +78,29 @@ const HIDDEN_ENVELOPE_TYPES = new Set([
'larpAutoInvite',
'karmaScore',
'pubAvailability',
'pam',
'calendarReminderSent',
'taskReminderSent',
'ubiClaim',
'ubiClaimResult',
'ubiRefuse',
'ubiAllocation',
'bankClaim',
'courts-key'
]);
module.exports = ({ cooler, tribeCrypto, tribesModel, padsModel, industryModel }) => {
module.exports = ({ cooler, tribeCrypto, tribesModel, padsModel, industryModel, larpModel }) => {
let ssb;
const openSsb = async () => { if (!ssb) ssb = await cooler.open(); return ssb };
let _feedCache = null;
let _feedCacheInflight = null;
const logTip = async () => {
try {
const ssbClient = await openSsb();
return await require('./typed_log').logTip(ssbClient);
} catch (_) { return null; }
};
const FEED_CACHE_MS = 15 * 1000;
const buildAccessibleTribeIds = async () => {
@ -95,14 +108,14 @@ module.exports = ({ cooler, tribeCrypto, tribesModel, padsModel, industryModel }
if (!tribesModel) return set;
try {
const list = await tribesModel.listAll();
for (const t of list) {
if (!t || !t.id) continue;
await Promise.all(list.map(async (t) => {
if (!t || !t.id) return;
set.add(t.id);
try {
const chain = await tribesModel.getChainIds(t.id);
for (const cid of chain) set.add(cid);
} catch (_) {}
}
}));
} catch (_) {}
return set;
};
@ -143,9 +156,6 @@ module.exports = ({ cooler, tribeCrypto, tribesModel, padsModel, industryModel }
return t.length > max ? t.slice(0, max - 1) + '…' : t;
};
// Builds self-contained chatThread items (title + recent messages embedded) for
// general (non-tribe) OPEN chats that have activity. Emitted as one item per chat
// with ts = latest message, so it survives the recent/all/mine filters intact.
const CHAT_THREAD_LIMIT = 6;
const buildChatThreads = async (ssbClient, idToAction, rootOf) => {
const replacedChatIds = new Set();
@ -160,28 +170,31 @@ module.exports = ({ cooler, tribeCrypto, tribesModel, padsModel, industryModel }
const root = rootOf(a.id);
const isTip = !replacedChatIds.has(a.id);
const prev = stateByRoot.get(root);
// Resolve the chain tip by replaces (a non-replaced action wins), breaking ts
// ties correctly — the mock and real SSB can stamp equal timestamps.
if (prev && !((isTip && !prev.isTip) || (isTip === prev.isTip && (a.ts || 0) >= prev.ts))) continue;
const cc = a.content || {};
stateByRoot.set(root, { ts: a.ts || 0, isTip, status: String(cc.status || '').toUpperCase(), tribeId: cc.tribeId || null, title: String(cc.title || ''), description: String(cc.description || ''), members: Array.isArray(cc.members) ? cc.members.length : 0 });
stateByRoot.set(root, { ts: a.ts || 0, isTip, owner: cc.author || a.author || null, status: String(cc.status || '').toUpperCase(), tribeId: cc.tribeId || null, title: String(cc.title || ''), description: String(cc.description || ''), members: Array.isArray(cc.members) ? cc.members.length : 0 });
}
const msgsByRoot = new Map();
for (const a of idToAction.values()) {
if (a.type !== 'chatMessage') continue;
const isPoll = a.type === 'poll';
if (a.type !== 'chatMessage' && !isPoll) continue;
const c = a.content || {};
if (c.tribeId || c.encryptedText) continue;
if (c.tribeId || c.encryptedText || c.encryptedQuestion) continue;
const root = c.chatId;
if (!root || typeof c.text !== 'string' || !c.text) continue;
if (!root) continue;
if (isPoll) {
if (!(typeof c.question === 'string' && c.question.trim())) continue;
} else if (!(typeof c.text === 'string' && c.text) && !c.image) continue;
if (!msgsByRoot.has(root)) msgsByRoot.set(root, []);
msgsByRoot.get(root).push(a);
msgsByRoot.get(root).push(isPoll ? { ...a, content: { chatId: root, text: c.question.trim(), poll: true } } : a);
}
for (const root of msgsByRoot.keys()) {
if (stateByRoot.has(root)) continue;
const val = await getMsg(ssbClient, root);
const missingRoots = [...msgsByRoot.keys()].filter(root => !stateByRoot.has(root));
const rootMsgs = new Map(await Promise.all(missingRoots.map(async root => [root, await getMsg(ssbClient, root)])));
for (const root of missingRoots) {
const val = rootMsgs.get(root);
const cc = val && val.content;
if (cc && typeof cc === 'object' && cc.type === 'chat') {
stateByRoot.set(root, { ts: 0, status: String(cc.status || '').toUpperCase(), tribeId: cc.tribeId || null, title: String(cc.title || ''), description: String(cc.description || ''), members: Array.isArray(cc.members) ? cc.members.length : 0 });
stateByRoot.set(root, { ts: 0, owner: cc.author || val.author || null, status: String(cc.status || '').toUpperCase(), tribeId: cc.tribeId || null, title: String(cc.title || ''), description: String(cc.description || ''), members: Array.isArray(cc.members) ? cc.members.length : 0 });
} else {
stateByRoot.set(root, null);
}
@ -200,11 +213,29 @@ module.exports = ({ cooler, tribeCrypto, tribesModel, padsModel, industryModel }
content: {
type: 'chatThread',
chatRoot: root,
chatAuthor: info.owner || null,
title: info.title || '',
description: info.description || '',
members: info.members || 0,
members: Math.max(info.members || 0, new Set(asc.map(m => m.author)).size),
messageCount: asc.length,
replies: asc.slice(-CHAT_THREAD_LIMIT).map(m => ({ id: m.id, author: m.author, ts: m.ts || 0, text: (m.content && m.content.text) || '' }))
replies: (() => {
const byId = new Map(asc.map(m => [m.id, m]));
return asc.slice(-CHAT_THREAD_LIMIT).map(m => {
const quotedId = (m.content && typeof m.content.replyTo === 'string') ? m.content.replyTo : null;
const quoted = quotedId ? byId.get(quotedId) : null;
return {
id: m.id,
author: m.author,
ts: m.ts || 0,
text: (m.content && m.content.text) || '',
poll: !!(m.content && m.content.poll),
image: (m.content && m.content.image) || null,
mimeType: (m.content && m.content.mimeType) || '',
replyTo: quotedId,
reply: quoted ? { id: quoted.id, author: quoted.author, text: String((quoted.content && quoted.content.text) || '').slice(0, 140) } : null
};
});
})()
}
});
}
@ -224,16 +255,18 @@ module.exports = ({ cooler, tribeCrypto, tribesModel, padsModel, industryModel }
if (entry && now - entry.ts < FEED_CACHE_MS) return entry.value;
if (!_feedCacheInflight) _feedCacheInflight = new Map();
if (_feedCacheInflight.has(cacheKey)) return _feedCacheInflight.get(cacheKey);
const tipNow = await logTip();
if (entry && tipNow && entry.tip === tipNow) { entry.ts = now; return entry.value; }
const promise = (async () => {
const ssbClient = await openSsb();
const userId = ssbClient.id;
const results = await new Promise((resolve, reject) => {
const results = (await new Promise((resolve, reject) => {
pull(
ssbClient.createLogStream({ reverse: true, limit: logLimit }),
ssbClient.createLogStream({}),
pull.collect((err, msgs) => err ? reject(err) : resolve(msgs))
);
});
})).reverse();
if (typeof ssbClient.createUserStream === 'function') {
const ownResults = await new Promise((resolve) => {
@ -249,6 +282,7 @@ module.exports = ({ cooler, tribeCrypto, tribesModel, padsModel, industryModel }
}
const tombstoned = buildValidatedTombstoneSet(results);
const chunkLookup = longText.lookupIn(longText.indexChunks(results));
const parentOf = new Map();
const idToAction = new Map();
const rawById = new Map();
@ -270,6 +304,20 @@ module.exports = ({ cooler, tribeCrypto, tribesModel, padsModel, industryModel }
}
const fpIdx = tribeCrypto ? tribeCrypto.buildFingerprintIndex() : null;
const accessibleTribeIds = await buildAccessibleTribeIds();
let larpWall = null;
if (larpModel && results.some(m => m && m.value && m.value.content && m.value.content.type === 'larpHousePost')) {
try {
const s = await openSsb();
larpWall = {
mine: await larpModel.getUserHouse(s.id).catch(() => null),
governing: typeof larpModel.getGoverningHouseKey === 'function' ? larpModel.getGoverningHouseKey() : null,
members: await larpModel.listAllMemberships().catch(() => new Map())
};
} catch (_) { larpWall = null; }
}
const larpPostVisible = (author, house) => !!larpWall && !!house
&& (larpWall.members.get(author) || 'academia') === house
&& (house === 'academia' || house === larpWall.governing || house === larpWall.mine);
for (const msg of results) {
const k = msg.key;
@ -277,7 +325,9 @@ module.exports = ({ cooler, tribeCrypto, tribesModel, padsModel, industryModel }
let c = v?.content;
if (!c) continue;
if (typeof c === 'string' && c.endsWith('.box')) continue;
if (c.private === true) continue;
if (c.type && HIDDEN_ENVELOPE_TYPES.has(c.type)) continue;
if (c.type === 'larpHousePost' && !larpPostVisible(v.author, String(c.house || '').toLowerCase())) continue;
if (typeof c.type === 'string' && /e2ee/i.test(c.type)) continue;
if (c.type === 'contact') continue;
if (tribeCrypto && tribeCrypto.isTribeMsg(c)) {
@ -307,6 +357,8 @@ module.exports = ({ cooler, tribeCrypto, tribesModel, padsModel, industryModel }
} catch (_) {}
}
if (!isContentSane(c)) continue;
if ((c.type === 'post' || c.type === 'forum') && longText.hasChunks(c)) c = longText.resolveField(c, 'text', v?.author, chunkLookup);
if (c.type === 'about' && !c.name && !c.description && !c.image) continue;
const ts = v?.timestamp || Number(c?.timestamp || 0) || (c?.updatedAt ? Date.parse(c.updatedAt) : 0) || 0;
idToAction.set(k, { id: k, author: v?.author, ts, type: inferType(c), content: c });
rawById.set(k, msg);
@ -315,17 +367,23 @@ module.exports = ({ cooler, tribeCrypto, tribesModel, padsModel, industryModel }
const forgedParentOf = new Map();
const forgedFeedContent = new Map();
const vg = versionGroups(Array.from(idToAction.values()).map(a => versionEntry({ key: a.id, value: { ...(rawById.get(a.id) || {}).value, content: a.content } })));
for (const [child, parent] of Array.from(parentOf.entries())) {
const ca = idToAction.get(child);
const pa = idToAction.get(parent);
if (!pa) { parentOf.delete(child); continue; }
if (!ca || String(ca.author) !== String(pa.author)) {
if (vg.isOrphan(child)) {
parentOf.delete(child);
if (vg.isForeignCopy(child)) { idToAction.delete(child); rawById.delete(child); }
continue;
}
if (!ca || vg.isForeign(child)) {
forgedParentOf.set(child, parent);
if (ca && ca.type === 'feed') forgedFeedContent.set(child, ca.content || {});
parentOf.delete(child);
idToAction.delete(child);
rawById.delete(child);
continue;
}
if (vg.isForeign(parent)) parentOf.set(child, vg.rootOf(child));
}
const replacedIds = new Set(parentOf.values());
@ -381,6 +439,10 @@ module.exports = ({ cooler, tribeCrypto, tribesModel, padsModel, industryModel }
}
const fetchedTargetCache = new Map();
const missingTargets = [...new Set([...idToAction.values()].filter(a => a.type === 'spread').map(a => (a.content || {}).vote?.link || '').filter(link => link && !rawById.get(link)))];
for (const [link, got] of await Promise.all(missingTargets.map(async link => [link, await getMsg(ssbClient, link)]))) {
fetchedTargetCache.set(link, got ? { key: link, value: got } : null);
}
for (const a of idToAction.values()) {
if (a.type !== 'spread') continue;
@ -540,7 +602,7 @@ module.exports = ({ cooler, tribeCrypto, tribesModel, padsModel, industryModel }
const v = msg && msg.value;
const c = v && v.content;
if (!c || typeof c !== 'object') continue;
const isOpinion = c.type === 'feedOpinion' && typeof c.target === 'string';
const isOpinion = typeof c.type === 'string' && /Opinion$/.test(c.type) && c.type !== 'pollOpinion' && typeof c.target === 'string';
const isVoteAction = c.type === 'feed-action' && c.action === 'vote' && (typeof c.root === 'string' || typeof c.target === 'string');
if (!isOpinion && !isVoteAction) continue;
const target = isOpinion ? c.target : String(c.root || c.target);
@ -570,6 +632,8 @@ module.exports = ({ cooler, tribeCrypto, tribesModel, padsModel, industryModel }
}
const latest = [];
const docUrls = [...new Set([...idToAction.values()].filter(a => a.type === 'document').map(a => (a.content || {}).url))];
const docBlobOk = new Map(await Promise.all(docUrls.map(async url => [url, await hasBlob(ssbClient, url)])));
for (const a of idToAction.values()) {
if (tombstoned.has(a.id)) continue;
if (a.type === 'tribe' && parentOf.has(a.id)) continue;
@ -580,11 +644,7 @@ module.exports = ({ cooler, tribeCrypto, tribesModel, padsModel, industryModel }
if (c.key && tombstoned.has(c.key)) continue;
if (c.branch && tombstoned.has(c.branch)) continue;
if (c.target && tombstoned.has(c.target)) continue;
if (a.type === 'document') {
const url = c.url;
const ok = await hasBlob(ssbClient, url);
if (!ok) continue;
}
if (a.type === 'document' && !docBlobOk.get(c.url)) continue;
if (a.type === 'forum' && c.root) {
const rootId = typeof c.root === 'string' ? c.root : (c.root?.key || c.root?.id || '');
const rootAction = idToAction.get(rootId);
@ -600,6 +660,19 @@ module.exports = ({ cooler, tribeCrypto, tribesModel, padsModel, industryModel }
a.pollVoters = tally ? tally.voters.size : 0;
}
let content = voteAgg ? { ...c, ...voteAgg } : a.content;
if (a.type !== 'feed' && a.type !== 'votes' && (feedOpinionsByRoot.get(actionRoot) || []).length) {
const base = a.content || {};
const opinions = { ...(base.opinions || {}) };
const voters = Array.isArray(base.opinions_inhabitants) ? base.opinions_inhabitants.slice() : [];
const voterSet = new Set(voters);
for (const op of feedOpinionsByRoot.get(actionRoot)) {
if (!op.author || voterSet.has(op.author)) continue;
voterSet.add(op.author);
voters.push(op.author);
if (op.category) opinions[op.category] = (Number(opinions[op.category]) || 0) + 1;
}
content = { ...content, opinions, opinions_inhabitants: voters };
}
if (a.type === 'feed') {
const base = a.content || {};
const opinions = { ...(base.opinions || {}) };
@ -629,6 +702,8 @@ module.exports = ({ cooler, tribeCrypto, tribesModel, padsModel, industryModel }
let deduped = latest.filter(a => !a.tipId || a.tipId === a.id || (a.type === 'tribe' && !parentOf.has(a.id)));
const mediaTypes = new Set(['image','video','audio','document','bookmark','map']);
const textTypes = new Set(['feed', 'post']);
const REPEAT_WINDOW_MS = 10 * 60 * 1000;
const perAuthorUnique = new Set();
const byKey = new Map();
const norm = s => String(s || '').trim().toLowerCase();
@ -668,6 +743,19 @@ module.exports = ({ cooler, tribeCrypto, tribesModel, padsModel, industryModel }
} else if (prevHasImage === newHasImage && effTs > prev.__effTs) {
byKey.set(key, { ...a, __effTs: effTs, __hasImage: newHasImage });
}
} else if (textTypes.has(a.type)) {
const t = norm(c.text);
const key = t ? `${a.type}:${a.author}:${t}` : null;
const prev = key ? byKey.get(key) : null;
if (!key) {
byKey.set(`id:${a.id}`, { ...a, __effTs: effTs });
} else if (!prev) {
byKey.set(key, { ...a, __effTs: effTs });
} else if (Math.abs(effTs - prev.__effTs) <= REPEAT_WINDOW_MS) {
if (effTs > prev.__effTs) byKey.set(key, { ...a, __effTs: effTs });
} else {
byKey.set(`id:${a.id}`, { ...a, __effTs: effTs });
}
} else if (a.type === 'tribe') {
const t = norm(c.title);
if (t) {
@ -710,7 +798,7 @@ module.exports = ({ cooler, tribeCrypto, tribesModel, padsModel, industryModel }
}
const tribeInternalTypes = new Set(['tribe-content', 'tribeParliamentCandidature', 'tribeParliamentTerm', 'tribeParliamentProposal', 'tribeParliamentRule', 'tribeParliamentLaw', 'tribeParliamentRevocation']);
const hiddenTypes = new Set(['padEntry', 'chatMessage', 'calendarDate', 'calendarNote', 'calendarReminderSent', 'taskReminderSent', 'feed-action', 'pubBalance', 'pubAvailability', 'log', 'logPublic', 'gameScore', 'pollVote', 'pollClose', 'pollOpinion', 'curriculum']);
const hiddenTypes = new Set(['wallet', 'bankWallet', 'padEntry', 'roomMember', 'chatMessage', 'calendarDate', 'calendarNote', 'calendarReminderSent', 'taskReminderSent', 'feed-action', 'pubBalance', 'pubAvailability', 'log', 'logPublic', 'gameScore', 'pollVote', 'pollClose', 'pollOpinion', 'curriculum', 'schoolLesson', 'schoolEnroll', 'schoolCertificate', 'schoolProgress', 'schoolExam', 'schoolExamResult', 'school-invite', 'schoolExamQuestion', 'schoolLessonMedia']);
const chatThreadItems = await buildChatThreads(ssbClient, idToAction, rootOf, deduped);
deduped = deduped.concat(chatThreadItems);
const isAllowedTribeActivity = (a) => {
@ -732,14 +820,18 @@ module.exports = ({ cooler, tribeCrypto, tribesModel, padsModel, industryModel }
const c = a.content || {};
if (c.encryptedPayload) return false;
if (a.type === 'pad' && c.status !== 'OPEN') return false;
if (a.type === 'room' && (c.status !== 'OPEN' || c.encrypted === true)) return false;
if (a.type === 'chat' && c.status !== 'OPEN') return false;
if (a.type === 'calendar' && c.status !== 'OPEN') return false;
if (a.type === 'event' && String(c.isPublic || '').toLowerCase() === 'private' && c.organizer !== userId && !(Array.isArray(c.attendees) && c.attendees.includes(userId))) return false;
if (a.type === 'task' && String(c.isPublic || '').toUpperCase() === 'PRIVATE' && a.author !== userId && !(Array.isArray(c.assignees) && c.assignees.includes(userId))) return false;
if (a.type === 'forum' && c.isPrivate === true && a.author !== userId) return false;
if (a.type === 'job' && String(c.visibility || '').toUpperCase() === 'HIDDEN' && a.author !== userId && !(Array.isArray(c.subscribers) && c.subscribers.includes(userId))) return false;
if (a.type === 'schoolCourse' && c.encryptedPayload) return false;
if (a.type === 'schoolCourse' && String(c.visibility || '').toUpperCase() === 'INVITE' && a.author !== userId && !(Array.isArray(c.invited) && c.invited.includes(userId))) return false;
if (a.type === 'housing' && String(c.visibility || '').toUpperCase() === 'HIDDEN' && a.author !== userId) return false;
if (a.type === 'market' && String(c.visibility || '').toUpperCase() === 'HIDDEN' && c.seller !== userId) return false;
if (a.type === 'market' && c.shopProductId) return false;
if (a.type === 'shop' && String(c.visibility || '').toUpperCase() === 'CLOSED' && a.author !== userId) return false;
if (a.type === 'curriculum' && String(c.visibility || '').toUpperCase() === 'HIDDEN' && a.author !== userId) return false;
if (a.type === 'shopProduct' && c.shopVisibility && String(c.shopVisibility).toUpperCase() === 'CLOSED' && a.author !== userId) return false;
@ -774,7 +866,6 @@ module.exports = ({ cooler, tribeCrypto, tribesModel, padsModel, industryModel }
.map(a => ({ ...a, authorActions: perAuthor.get(a.author) || 0 }))
.sort((x, y) => (y.authorActions - x.authorActions) || ((y.ts || 0) - (x.ts || 0)));
}
else if (filter === 'banking') out = deduped.filter(a => a.type === 'bankWallet' || a.type === 'bankClaim' || a.type === 'ubiClaim');
else if (filter === 'tribe') out = deduped.filter(a => a.type === 'tribe' || String(a.type || '').startsWith('tribe'));
else if (filter === 'spread') out = deduped.filter(a => a.type === 'spread');
else if (filter === 'parliament')
@ -793,6 +884,13 @@ module.exports = ({ cooler, tribeCrypto, tribesModel, padsModel, industryModel }
else if (filter === 'industry')
out = deduped.filter(a => ['industry', 'industryBuild', 'industryBlueprint', 'industryAllocation'].includes(a.type) && isVisible(a));
else if (filter === 'pad') out = deduped.filter(a => a.type === 'pad' && (a.content || {}).status === 'OPEN');
else if (filter === 'room') out = deduped.filter(a => a.type === 'room' && (a.content || {}).status === 'OPEN' && (a.content || {}).encrypted !== true);
else if (filter === 'wiki') out = deduped.filter(a => a.type === 'wikiPage' && isVisible(a));
else if (filter === 'emergency') out = deduped.filter(a => (a.type === 'emergency' || a.type === 'emergencyUpdate') && isVisible(a));
else if (filter === 'campaign') out = deduped.filter(a => (a.type === 'campaign' || a.type === 'campaignUpdate') && isVisible(a));
else if (filter === 'mailing') out = deduped.filter(a => a.type === 'mailingList' && isVisible(a));
else if (filter === 'logistics') out = deduped.filter(a => a.type === 'logisticsRoute' && isVisible(a));
else if (filter === 'podcast') out = deduped.filter(a => (a.type === 'podcast' || a.type === 'podcastEpisode') && isVisible(a));
else if (filter === 'chat') out = deduped.filter(a => (a.type === 'chat' || a.type === 'chatThread') && isAllowedTribeActivity(a) && isVisible(a));
else if (filter === 'calendar') out = deduped.filter(a => a.type === 'calendar' && (a.content || {}).status === 'OPEN');
else if (filter === 'transfer') out = deduped.filter(a => a.type === 'transfer' && isVisible(a));
@ -804,7 +902,7 @@ module.exports = ({ cooler, tribeCrypto, tribesModel, padsModel, industryModel }
_feedCacheInflight.set(cacheKey, promise);
try {
const value = await promise;
_feedCache.set(cacheKey, { value, ts: Date.now() });
_feedCache.set(cacheKey, { value, ts: Date.now(), tip: tipNow });
return value;
} finally {
_feedCacheInflight.delete(cacheKey);

254
src/models/agenda_model.js
View file

@ -3,9 +3,11 @@ const path = require('path');
const pull = require('../server/node_modules/pull-stream');
const moment = require('../server/node_modules/moment');
const agendaConfigPath = path.join(__dirname, '../configs/agenda-config.json');
const agendaConfigPath = require('../configs/state-manager').statePath('agenda-config.json');
const { getConfig } = require('../configs/config-manager.js');
const { buildValidatedTombstoneSet } = require('./tombstone_validator');
const { readTyped } = require('./typed_log');
const { versionGroups, versionEntry } = require('../backend/dedupe');
const logLimit = getConfig().ssbLogStream?.limit || 1000;
function readAgendaConfig() {
@ -19,137 +21,117 @@ function writeAgendaConfig(cfg) {
fs.writeFileSync(agendaConfigPath, JSON.stringify(cfg, null, 2));
}
module.exports = ({ cooler, calendarsModel, eventsModel, tasksModel, marketModel, jobsModel, projectsModel, industryModel, housingModel }) => {
module.exports = ({ cooler, campaignsModel = null, logisticsModel = null, calendarsModel, eventsModel, tasksModel, marketModel, jobsModel, projectsModel, industryModel, housingModel, schoolModel }) => {
let ssb;
const openSsb = async () => { if (!ssb) ssb = await cooler.open(); return ssb; };
const STATUS_ORDER = ['FOR SALE', 'OPEN', 'RESERVED', 'CLOSED', 'SOLD'];
const sIdx = s => STATUS_ORDER.indexOf(String(s || '').toUpperCase());
const fetchItems = (targetType) =>
new Promise((resolve, reject) => {
openSsb().then((ssbClient) => {
pull(
ssbClient.createLogStream({ limit: logLimit }),
pull.collect((err, msgs) => {
if (err) return reject(err);
const fetchItems = async (targetType) => {
const ssbClient = await openSsb();
const msgs = await readTyped(ssbClient, [targetType, 'tombstone'], { limit: logLimit });
const tomb = buildValidatedTombstoneSet(msgs);
const nodes = new Map();
const parent = new Map();
const child = new Map();
const tomb = buildValidatedTombstoneSet(msgs);
const nodes = new Map();
const versions = [];
for (const m of msgs) {
const k = m.key;
const v = m.value;
const c = v?.content;
if (!c) continue;
if (c.type === 'tombstone') continue;
if (c.type !== targetType) continue;
if (c.encryptedPayload) continue;
nodes.set(k, { key: k, ts: v.timestamp || 0, author: v.author, content: c });
if (c.replaces) parent.set(k, c.replaces);
}
for (const m of msgs) {
const k = m.key;
const v = m.value;
const c = v?.content;
if (!c) continue;
if (c.type === 'tombstone') continue;
if (c.type !== targetType) continue;
if (c.encryptedPayload) continue;
nodes.set(k, { key: k, ts: v.timestamp || 0, author: v.author, content: c });
versions.push(versionEntry(m));
}
for (const [k, p] of Array.from(parent.entries())) {
const cn = nodes.get(k);
const pn = nodes.get(p);
if (!pn) { parent.delete(k); continue; }
if (!cn || String(cn.author) !== String(pn.author)) { parent.delete(k); nodes.delete(k); }
}
for (const [k, p] of parent.entries()) child.set(p, k);
const vg = versionGroups(versions);
const groups = new Map();
for (const r of vg.roots) groups.set(r, new Set(vg.groups.get(r).filter(id => !vg.isForeign(id))));
const rootOf = (id) => { let cur = id; while (parent.has(cur) && nodes.has(parent.get(cur))) cur = parent.get(cur); return cur; };
const statusOrder = ['FOR SALE', 'OPEN', 'RESERVED', 'CLOSED', 'SOLD'];
const sIdx = s => statusOrder.indexOf(String(s || '').toUpperCase());
const groups = new Map();
for (const id of nodes.keys()) {
const r = rootOf(id);
if (!groups.has(r)) groups.set(r, new Set());
groups.get(r).add(id);
}
const out = [];
const statusOrder = ['FOR SALE', 'OPEN', 'RESERVED', 'CLOSED', 'SOLD'];
const sIdx = s => statusOrder.indexOf(String(s || '').toUpperCase());
for (const [root, ids] of groups.entries()) {
const items = Array.from(ids).map(id => nodes.get(id)).filter(n => n && !tomb.has(n.key));
if (!items.length) continue;
const out = [];
const tip = nodes.get(vg.tipOf(root));
if (tomb.has(tip.key)) continue;
for (const [root, ids] of groups.entries()) {
const items = Array.from(ids).map(id => nodes.get(id)).filter(n => n && !tomb.has(n.key));
if (!items.length) continue;
if (targetType === 'market') {
let chosen = items[0];
for (const n of items) {
const a = sIdx(n.content.status);
const b = sIdx(chosen.content.status);
if (a > b || (a === b && n.ts > chosen.ts)) chosen = n;
}
const c = chosen.content;
let status = c.status;
if (c.deadline) {
const dl = moment(c.deadline);
if (dl.isValid() && dl.isBefore(moment()) && String(status).toUpperCase() !== 'SOLD') status = 'DISCARDED';
}
if (status === 'FOR SALE' && (c.stock || 0) === 0) continue;
let tipId = Array.from(ids).find(id => !child.has(id));
let tip = tipId ? nodes.get(tipId) : items.reduce((a, b) => a.ts > b.ts ? a : b);
out.push({
...c,
status,
author: chosen.author,
id: chosen.key,
tipId: chosen.key,
createdAt: c.createdAt || chosen.ts
});
continue;
}
if (targetType === 'market') {
let chosen = items[0];
for (const n of items) {
const a = sIdx(n.content.status);
const b = sIdx(chosen.content.status);
if (a > b || (a === b && n.ts > chosen.ts)) chosen = n;
}
const c = chosen.content;
let status = c.status;
if (c.deadline) {
const dl = moment(c.deadline);
if (dl.isValid() && dl.isBefore(moment()) && String(status).toUpperCase() !== 'SOLD') status = 'DISCARDED';
}
if (status === 'FOR SALE' && (c.stock || 0) === 0) continue;
if (targetType === 'job') {
const latest = items.sort((a, b) => b.ts - a.ts)[0];
const withSubsNode = items
.filter(n => Array.isArray(n.content.subscribers))
.sort((a, b) => b.ts - a.ts)[0];
const subscribers = withSubsNode ? withSubsNode.content.subscribers : [];
const latestWithStatus = items
.filter(n => typeof n.content.status !== 'undefined')
.sort((a, b) => b.ts - a.ts)[0];
const resolvedStatus = latestWithStatus
? latestWithStatus.content.status
: latest.content.status;
out.push({
...c,
status,
author: chosen.author,
id: chosen.key,
tipId: chosen.key,
createdAt: c.createdAt || chosen.ts
});
continue;
}
const c = { ...latest.content, status: resolvedStatus, subscribers };
if (targetType === 'job') {
const latest = items.sort((a, b) => b.ts - a.ts)[0];
const withSubsNode = items
.filter(n => Array.isArray(n.content.subscribers))
.sort((a, b) => b.ts - a.ts)[0];
const subscribers = withSubsNode ? withSubsNode.content.subscribers : [];
const latestWithStatus = items
.filter(n => typeof n.content.status !== 'undefined')
.sort((a, b) => b.ts - a.ts)[0];
const resolvedStatus = latestWithStatus
? latestWithStatus.content.status
: latest.content.status;
out.push({
...c,
author: latest.author,
id: latest.key,
tipId: latest.key,
createdAt: c.createdAt || latest.ts
});
continue;
}
const c = { ...latest.content, status: resolvedStatus, subscribers };
out.push({
...tip.content,
author: tip.author,
id: tip.key,
tipId: tip.key,
createdAt: tip.content.createdAt || tip.ts
});
}
out.push({
...c,
author: latest.author,
id: latest.key,
tipId: latest.key,
createdAt: c.createdAt || latest.ts
});
continue;
}
out.push({
...tip.content,
author: tip.author,
id: tip.key,
tipId: tip.key,
createdAt: tip.content.createdAt || tip.ts
});
}
resolve(out);
})
);
}).catch(reject);
});
return out;
};
return {
async listAgenda(filter = 'all') {
const agendaConfig = readAgendaConfig();
const discardedItems = agendaConfig.discardedItems || [];
const removedItems = agendaConfig.removedItems || [];
const ssbClient = await openSsb();
const userId = ssbClient.id;
@ -192,7 +174,19 @@ module.exports = ({ cooler, calendarsModel, eventsModel, tasksModel, marketModel
? housingModel.listHousing('ALL', userId).then(normalize).catch(() => [])
: [];
const [tasksAll, eventsAll, transfersAll, tribesAll, marketAll, reportsAll, jobsAll, projectsAll, calendarsAll, industryAll, housingAll] = await Promise.all([
const schoolViaModel = schoolModel && typeof schoolModel.listCourses === 'function'
? schoolModel.listCourses('ALL', userId, {}).then(normalize).catch(() => [])
: [];
const campaignsViaModel = campaignsModel && typeof campaignsModel.listAll === 'function'
? campaignsModel.listAll({ filter: 'all' }).catch(() => [])
: [];
const logisticsViaModel = logisticsModel && typeof logisticsModel.listAll === 'function'
? logisticsModel.listAll({ filter: 'all' }).catch(() => [])
: [];
const [tasksAll, eventsAll, transfersAll, tribesAll, marketAll, reportsAll, jobsAll, projectsAll, calendarsAll, industryAll, housingAll, schoolAll, campaignsAll, logisticsAll] = await Promise.all([
tasksViaModel,
eventsViaModel,
fetchItems('transfer'),
@ -203,7 +197,10 @@ module.exports = ({ cooler, calendarsModel, eventsModel, tasksModel, marketModel
projectsViaModel,
calendarsViaModel,
industryViaModel,
housingViaModel
housingViaModel,
schoolViaModel,
campaignsViaModel,
logisticsViaModel
]);
const tasks = tasksAll.filter(c => Array.isArray(c.assignees) && c.assignees.includes(userId)).map(t => ({ ...t, type: 'task' }));
@ -215,26 +212,36 @@ module.exports = ({ cooler, calendarsModel, eventsModel, tasksModel, marketModel
).map(m => ({ ...m, type: 'market' }));
const reports = reportsAll.filter(c => c.author === userId || (Array.isArray(c.confirmations) && c.confirmations.includes(userId))).map(r => ({ ...r, type: 'report' }));
const jobs = jobsAll.filter(c => c.author === userId || (Array.isArray(c.subscribers) && c.subscribers.includes(userId))).map(j => ({ ...j, type: 'job', title: j.title }));
const projects = projectsAll.map(p => ({ ...p, type: 'project' }));
const projects = projectsAll.filter(p => p.author === userId || (Array.isArray(p.followers) && p.followers.includes(userId)) || (Array.isArray(p.backers) && p.backers.some(b => b && b.userId === userId)) || (Array.isArray(p.bounties) && p.bounties.some(b => b && b.claimedBy === userId))).map(p => ({ ...p, type: 'project' }));
const industryBuilds = industryAll.map(b => ({ ...b, type: 'industry', title: b.title }));
const schoolCourses = schoolAll
.filter(c => c.author === userId || (Array.isArray(c.students) && c.students.includes(userId)) || (Array.isArray(c.pending) && c.pending.some(pn => pn.author === userId)))
.map(c => ({ ...c, type: 'schoolCourse', title: c.title, date: c.startDate || c.createdAt, teaching: c.author === userId }));
const campaigns = (Array.isArray(campaignsAll) ? campaignsAll : [])
.filter(c => c && c.deadline && (c.author === userId || (Array.isArray(c.signers) && c.signers.includes(userId))))
.map(c => ({ ...c, type: 'campaign', date: c.deadline, status: c.closed ? 'CLOSED' : 'OPEN' }));
const routes = (Array.isArray(logisticsAll) ? logisticsAll : [])
.filter(r => r && r.date && (r.author === userId || !!r.myBooking))
.map(r => ({ ...r, type: 'logisticsRoute', status: r.status === 'CLOSED' ? 'CLOSED' : 'OPEN' }));
const housingPlaces = housingAll
.filter(h => h.author === userId || (Array.isArray(h.requests) && h.requests.includes(userId)))
.map(h => ({ ...h, type: 'housing', date: h.availableFrom || h.createdAt, requested: Array.isArray(h.requests) && h.requests.includes(userId) }));
const myCalendars = calendarsAll
.filter(c => c.author === userId || (Array.isArray(c.participants) && c.participants.includes(userId)));
const calendars = myCalendars.map(c => ({ ...c, type: 'calendar' }));
const calendars = myCalendars.map(c => ({ ...c, id: c.rootId || c.id || c.key, type: 'calendar' }));
const calendarDates = [];
if (calendarsModel && typeof calendarsModel.getDatesForCalendar === 'function') {
for (const cal of myCalendars) {
const datesByCal = await Promise.all(myCalendars.map(cal => calendarsModel.getDatesForCalendar(cal.rootId || cal.id || cal.key).catch(() => [])));
for (const [ci, cal] of myCalendars.entries()) {
try {
const dates = await calendarsModel.getDatesForCalendar(cal.id || cal.key);
const dates = datesByCal[ci];
for (const d of (dates || [])) {
calendarDates.push({
type: 'calendarDate',
id: d.key || d.id,
title: d.label || cal.title || 'Calendar date',
calendarTitle: cal.title || '',
calendarId: cal.id || cal.key,
calendarId: cal.rootId || cal.id || cal.key,
date: d.date || d.createdAt,
label: d.label || '',
author: cal.author,
@ -257,9 +264,12 @@ module.exports = ({ cooler, calendarsModel, eventsModel, tasksModel, marketModel
...projects,
...industryBuilds,
...housingPlaces,
...schoolCourses,
...campaigns,
...routes,
...calendars,
...calendarDates
];
].filter(i => !removedItems.includes(i.id));
let filtered;
if (filter === 'discarded') {
@ -278,6 +288,9 @@ module.exports = ({ cooler, calendarsModel, eventsModel, tasksModel, marketModel
else if (filter === 'projects') filtered = filtered.filter(i => i.type === 'project');
else if (filter === 'industry') filtered = filtered.filter(i => i.type === 'industry');
else if (filter === 'housing') filtered = filtered.filter(i => i.type === 'housing');
else if (filter === 'school') filtered = filtered.filter(i => i.type === 'schoolCourse');
else if (filter === 'campaigns') filtered = filtered.filter(i => i.type === 'campaign');
else if (filter === 'logistics') filtered = filtered.filter(i => i.type === 'logisticsRoute');
else if (filter === 'calendars') filtered = filtered.filter(i => i.type === 'calendar' || i.type === 'calendarDate');
else if (filter === 'today') {
const startOfDay = new Date(); startOfDay.setHours(0, 0, 0, 0);
@ -333,6 +346,9 @@ module.exports = ({ cooler, calendarsModel, eventsModel, tasksModel, marketModel
projects: mainItems.filter(i => i.type === 'project').length,
industry: mainItems.filter(i => i.type === 'industry').length,
housing: mainItems.filter(i => i.type === 'housing').length,
school: mainItems.filter(i => i.type === 'schoolCourse').length,
campaigns: mainItems.filter(i => i.type === 'campaign').length,
logistics: mainItems.filter(i => i.type === 'logisticsRoute').length,
calendars: mainItems.filter(i => i.type === 'calendar' || i.type === 'calendarDate').length,
today: mainItems.filter(i => { const d = itemTs(i); return d >= startOfDay.getTime() && d <= endOfDay.getTime(); }).length,
upcoming: mainItems.filter(i => itemTs(i) > now).length,
@ -354,6 +370,14 @@ module.exports = ({ cooler, calendarsModel, eventsModel, tasksModel, marketModel
const agendaConfig = readAgendaConfig();
agendaConfig.discardedItems = agendaConfig.discardedItems.filter(id => id !== itemId);
writeAgendaConfig(agendaConfig);
},
async removeItem(itemId) {
const agendaConfig = readAgendaConfig();
agendaConfig.discardedItems = (agendaConfig.discardedItems || []).filter(id => id !== itemId);
agendaConfig.removedItems = agendaConfig.removedItems || [];
if (!agendaConfig.removedItems.includes(itemId)) agendaConfig.removedItems.push(itemId);
writeAgendaConfig(agendaConfig);
}
};
};

75
src/models/audios_model.js
View file

@ -1,6 +1,7 @@
const pull = require("../server/node_modules/pull-stream");
const { buildValidatedTombstoneSet } = require('./tombstone_validator');
const { dedupeBy, norm } = require('../backend/dedupe');
const { dedupeBy, norm, versionGroups } = require('../backend/dedupe');
const { readTyped, memoIndex } = require('./typed_log');
const { getConfig } = require("../configs/config-manager.js");
const categories = require("../backend/opinion_categories");
@ -31,10 +32,9 @@ module.exports = ({ cooler }) => {
return ssb;
};
const getAllMessages = async (ssbClient) =>
new Promise((resolve, reject) => {
pull(ssbClient.createLogStream({ limit: logLimit }), pull.collect((err, msgs) => (err ? reject(err) : resolve(msgs))));
});
const AUDIO_TYPES = ["audio", "audioOpinion", "tombstone"];
const getAllMessages = async (ssbClient) => readTyped(ssbClient, AUDIO_TYPES, { limit: logLimit });
const getMsg = async (ssbClient, key) =>
new Promise((resolve, reject) => {
@ -44,9 +44,6 @@ module.exports = ({ cooler }) => {
const buildIndex = (messages) => {
const tomb = buildValidatedTombstoneSet(messages);
const nodes = new Map();
const parent = new Map();
const child = new Map();
const strictChild = new Map();
const opinionMsgs = [];
for (const m of messages) {
@ -62,41 +59,16 @@ module.exports = ({ cooler }) => {
const ts = v.timestamp || m.timestamp || 0;
let sizeBytes = 0;
try { sizeBytes = Buffer.byteLength(JSON.stringify(v), 'utf8'); } catch (_) { sizeBytes = 0; }
nodes.set(k, { key: k, ts, c, sizeBytes, author: v.author });
if (c.replaces) {
parent.set(k, c.replaces);
child.set(c.replaces, k);
}
nodes.set(k, { key: k, ts, seq: v.sequence || 0, c, sizeBytes, author: v.author });
}
for (const [k, node] of nodes) {
const t = node.c.replaces;
if (t) { const orig = nodes.get(t); if (orig && orig.author === node.author) strictChild.set(t, k); }
}
const rootOf = (id) => {
let cur = id, g = 0;
while (parent.has(cur) && nodes.has(parent.get(cur)) && g++ < 100000) cur = parent.get(cur);
return cur;
};
const contentTipOf = (root) => {
let cur = root, g = 0;
while (strictChild.has(cur) && g++ < 100000) cur = strictChild.get(cur);
const n = nodes.get(cur), rn = nodes.get(root);
return (n && rn && n.author === rn.author) ? cur : root;
};
const roots = new Set();
for (const id of nodes.keys()) roots.add(rootOf(id));
const vg = versionGroups([...nodes.values()].map(n => ({ key: n.key, author: n.author, seq: n.seq, ts: n.ts, replaces: n.c.replaces || null, claimedAuthor: typeof n.c.author === "string" ? n.c.author : null })));
const rootOf = vg.rootOf;
const contentTipOf = vg.tipOf;
const opinionsByRoot = new Map();
for (const op of opinionMsgs) { if (!nodes.has(op.target)) continue; const r = rootOf(op.target); if (!opinionsByRoot.has(r)) opinionsByRoot.set(r, []); opinionsByRoot.get(r).push(op); }
const forward = new Map();
for (const [newId, oldId] of parent.entries()) forward.set(oldId, newId);
const resolveGroup = (root) => {
const contentTip = contentTipOf(root);
const contentNode = nodes.get(contentTip) || nodes.get(root);
@ -113,9 +85,9 @@ module.exports = ({ cooler }) => {
};
const tipByRoot = new Map();
for (const r of roots) tipByRoot.set(r, contentTipOf(r));
for (const r of vg.roots) tipByRoot.set(r, contentTipOf(r));
return { tomb, nodes, parent, child, rootOf, tipByRoot, forward, resolveGroup };
return { tomb, nodes, rootOf, tipByRoot, resolveGroup };
};
const buildAudio = (node, rootId, viewerId, agg) => {
@ -132,9 +104,10 @@ module.exports = ({ cooler }) => {
createdAt: c.createdAt || new Date(node.ts).toISOString(),
updatedAt: c.updatedAt || null,
tags: tagsArr,
author: c.author,
author: node.author,
title: c.title || "",
description: c.description || "",
license: c.license || "",
mapUrl: c.mapUrl || "",
opinions,
opinions_inhabitants: voters,
@ -163,16 +136,12 @@ module.exports = ({ cooler }) => {
const messages = await getAllMessages(ssbClient);
const idx = buildIndex(messages);
let tip = id;
while (idx.forward.has(tip)) tip = idx.forward.get(tip);
if (idx.tomb.has(tip)) throw new Error("Audio not found");
let root = tip;
while (idx.parent.has(root)) root = idx.parent.get(root);
const root = idx.rootOf(id);
if (idx.tomb.has(idx.tipByRoot.get(root) || id)) throw new Error("Audio not found");
return root;
},
async createAudio(blobMarkdown, tagsRaw, title, description, mapUrl) {
async createAudio(blobMarkdown, tagsRaw, title, description, mapUrl, license) {
const ssbClient = await openSsb();
const blobId = parseBlobId(blobMarkdown);
const tags = normalizeTags(tagsRaw) || [];
@ -189,7 +158,8 @@ module.exports = ({ cooler }) => {
description: description || "",
mapUrl: mapUrl || "",
opinions: {},
opinions_inhabitants: []
opinions_inhabitants: [],
...(license ? { license } : {})
};
return new Promise((resolve, reject) => {
@ -251,7 +221,7 @@ module.exports = ({ cooler }) => {
});
},
async updateAudioById(id, blobMarkdown, tagsRaw, title, description, mapUrl) {
async updateAudioById(id, blobMarkdown, tagsRaw, title, description, mapUrl, license) {
const ssbClient = await openSsb();
const userId = ssbClient.id;
const tipId = await this.resolveCurrentId(id);
@ -275,7 +245,8 @@ module.exports = ({ cooler }) => {
description: description !== undefined ? description || "" : oldMsg.content.description || "",
mapUrl: mapUrl !== undefined ? mapUrl || "" : oldMsg.content.mapUrl || "",
createdAt: oldMsg.content.createdAt,
updatedAt: now
updatedAt: now,
...(license !== undefined ? { license } : {})
};
const tombstone = { type: "tombstone", target: tipId, deletedAt: now, author: userId };
@ -358,11 +329,11 @@ module.exports = ({ cooler }) => {
const ssbClient = await openSsb();
const viewer = viewerId || ssbClient.id;
const messages = await getAllMessages(ssbClient);
const idx = buildIndex(messages);
const idx = memoIndex('audios', messages, buildIndex);
const rootId = idx.rootOf(id);
const agg = idx.resolveGroup(rootId);
if (!agg.contentNode || idx.tomb.has(agg.contentTip)) throw new Error("Audio not found");
if (!idx.tipByRoot.has(rootId) || !agg.contentNode || idx.tomb.has(agg.contentTip)) throw new Error("Audio not found");
return buildAudio(agg.contentNode, rootId, viewer, agg);
},

779
src/models/backup_model.js Normal file
View file

@ -0,0 +1,779 @@
const fs = require('fs');
const os = require('os');
const path = require('path');
const zlib = require('zlib');
const crypto = require('crypto');
const { pipeline } = require('stream');
const pull = require('../server/node_modules/pull-stream');
const toPull = require('../server/node_modules/stream-to-pull-stream');
const KEY_MAGIC = Buffer.from('OASIS1');
const BACKUP_MAGIC = Buffer.from('OASISBK1');
const SNAPSHOT_MAGIC = Buffer.from('OASISSN1');
const SNAPSHOT_MAX_BYTES = 2 * 1024 * 1024 * 1024;
const SALT_LEN = 16;
const IV_LEN = 12;
const TAG_LEN = 16;
const REC_META = 1;
const REC_MSG = 2;
const REC_BLOB = 3;
const REC_FILE = 4;
const REC_MAX_LEN = 256 * 1024 * 1024;
const stateRoot = () => {
try { return path.join(require('../configs/state-manager').ssbDir(), 'oasis'); } catch (_) { return null; }
};
const NOT_BACKED_UP = new Set(['oasis-config.json', 'oasis-server-config.json', 'rooms']);
const listStateFiles = (dir, base = dir) => {
const out = [];
let entries = [];
try { entries = fs.readdirSync(dir, { withFileTypes: true }); } catch (_) { return out; }
for (const entry of entries) {
if (dir === base && NOT_BACKED_UP.has(entry.name)) continue;
const full = path.join(dir, entry.name);
if (entry.isDirectory()) out.push(...listStateFiles(full, base));
else if (entry.isFile() && !entry.name.endsWith('.migrated') && !/^snapshot(-recent)?\.oasissn/.test(entry.name)) out.push({ rel: path.relative(base, full), full });
}
return out;
};
const BLOB_RE = /&[A-Za-z0-9+/=]{44}\.sha256/g;
const DEFAULT_REMINDER_DAYS = 30;
const REMINDER_OPTIONS = [7, 30, 90, 0];
const MODULE_GROUPS = [
{ key: 'feed', test: (t) => /^(post|feed|about|contact|pub|vote|subscription)$/.test(t) },
{ key: 'media', test: (t) => /^(audio|video|image|document|bookmark|torrent|file)/i.test(t) },
{ key: 'blogs', test: (t) => /^blog/i.test(t) },
{ key: 'forum', test: (t) => /^forum/i.test(t) },
{ key: 'chats', test: (t) => /^chat/i.test(t) },
{ key: 'pads', test: (t) => /^pad/i.test(t) },
{ key: 'rooms', test: (t) => /^room/i.test(t) },
{ key: 'wiki', test: (t) => /^wiki/i.test(t) },
{ key: 'maps', test: (t) => /^map/i.test(t) },
{ key: 'tribes', test: (t) => /^tribe/i.test(t) },
{ key: 'school', test: (t) => /^school/i.test(t) },
{ key: 'parliament', test: (t) => /^parliament/i.test(t) },
{ key: 'courts', test: (t) => /^courts/i.test(t) },
{ key: 'emergencies', test: (t) => /^emergency/i.test(t) },
{ key: 'votes', test: (t) => /^(votes?|poll)/i.test(t) },
{ key: 'events', test: (t) => /^event/i.test(t) },
{ key: 'calendars', test: (t) => /^calendar/i.test(t) },
{ key: 'tasks', test: (t) => /^task/i.test(t) },
{ key: 'reports', test: (t) => /^report/i.test(t) },
{ key: 'mailing', test: (t) => /^mailing/i.test(t) },
{ key: 'campaigns', test: (t) => /^campaign/i.test(t) },
{ key: 'market', test: (t) => /^market/i.test(t) },
{ key: 'shops', test: (t) => /^shop/i.test(t) },
{ key: 'jobs', test: (t) => /^(job|curriculum)/i.test(t) },
{ key: 'housing', test: (t) => /^housing/i.test(t) },
{ key: 'projects', test: (t) => /^project/i.test(t) },
{ key: 'industry', test: (t) => /^industry/i.test(t) },
{ key: 'banking', test: (t) => /^(bank|ubi|wallet)/i.test(t) },
{ key: 'transfers', test: (t) => /^transfer/i.test(t) },
{ key: 'logistics', test: (t) => /^logistics/i.test(t) },
{ key: 'podcasts', test: (t) => /^podcast/i.test(t) },
{ key: 'games', test: (t) => /^(game|larp|pixelia|melody)/i.test(t) },
{ key: 'logs', test: (t) => /^log/i.test(t) }
];
const MODULE_KEYS = [...MODULE_GROUPS.map(g => g.key), 'inbox', 'other'];
const moduleOf = (content) => {
if (typeof content === 'string') return 'inbox';
const t = content && typeof content === 'object' ? String(content.type || '') : '';
if (!t) return 'other';
const g = MODULE_GROUPS.find(x => x.test(t));
return g ? g.key : 'other';
};
const normalizePassword = (password) => {
if (password && typeof password === 'object' && password.password) password = password.password;
return String(password || '');
};
const deriveKey = (password, salt) => crypto.scryptSync(password, salt, 32);
const encryptBuffer = (plaintext, password) => {
const salt = crypto.randomBytes(SALT_LEN);
const iv = crypto.randomBytes(IV_LEN);
const cipher = crypto.createCipheriv('aes-256-gcm', deriveKey(password, salt), iv);
const ciphertext = Buffer.concat([cipher.update(plaintext), cipher.final()]);
return Buffer.concat([KEY_MAGIC, salt, iv, cipher.getAuthTag(), ciphertext]);
};
const decryptBuffer = (data, password) => {
if (!Buffer.isBuffer(data) || data.length < KEY_MAGIC.length + SALT_LEN + IV_LEN + TAG_LEN) throw new Error('Unrecognized or corrupt backup file.');
if (!data.slice(0, KEY_MAGIC.length).equals(KEY_MAGIC)) throw new Error('Unrecognized or corrupt backup file.');
let o = KEY_MAGIC.length;
const salt = data.slice(o, o += SALT_LEN);
const iv = data.slice(o, o += IV_LEN);
const tag = data.slice(o, o += TAG_LEN);
const decipher = crypto.createDecipheriv('aes-256-gcm', deriveKey(password, salt), iv);
decipher.setAuthTag(tag);
return Buffer.concat([decipher.update(data.slice(o)), decipher.final()]);
};
const ssbDirOf = () => require('../server/ssb_config').path;
const secretPathOf = () => path.join(ssbDirOf(), 'secret');
const stateFile = () => require('../configs/state-manager').statePath('oasis-backup.json');
const readState = () => {
try { return JSON.parse(fs.readFileSync(stateFile(), 'utf8')) || {}; } catch (_) { return {}; }
};
const writeState = (next) => {
try {
const p = stateFile();
fs.mkdirSync(path.dirname(p), { recursive: true });
fs.writeFileSync(p, JSON.stringify(next, null, 2), { mode: 0o600 });
} catch (_) {}
return next;
};
const reminderState = () => {
const b = readState();
const days = b.reminderDays === undefined ? DEFAULT_REMINDER_DAYS : Number(b.reminderDays);
const lastAt = b.lastAt || null;
const lastTs = lastAt ? Date.parse(lastAt) || 0 : 0;
const overdue = days > 0 && (!lastTs || Date.now() - lastTs > days * 86400000);
return { lastAt, reminderDays: days, overdue, daysSince: lastTs ? Math.floor((Date.now() - lastTs) / 86400000) : null };
};
const setReminderDays = (days) => {
const n = Number(days);
return writeState({ ...readState(), reminderDays: REMINDER_OPTIONS.includes(n) ? n : DEFAULT_REMINDER_DAYS });
};
const markBackup = () => writeState({ ...readState(), lastAt: new Date().toISOString() });
const blobRefsOf = (value) => {
let json;
try { json = JSON.stringify(value); } catch (_) { return []; }
return Array.from(new Set(json.match(BLOB_RE) || []));
};
const record = (type, payload) => {
const head = Buffer.alloc(5);
head.writeUInt8(type, 0);
head.writeUInt32BE(payload.length, 1);
return Buffer.concat([head, payload]);
};
const writeChunk = (stream, chunk) => new Promise((resolve, reject) => {
const ok = stream.write(chunk, (err) => err ? reject(err) : null);
if (ok) resolve();
else stream.once('drain', resolve);
});
module.exports = ({ cooler }) => {
let ssb;
let restoreJob = null;
const openSsb = async () => { if (!ssb) ssb = await cooler.open(); return ssb; };
const readLog = async (ssbClient) => new Promise((resolve, reject) => {
pull(ssbClient.createLogStream({ reverse: false }), pull.collect((err, arr) => err ? reject(err) : resolve(arr || [])));
});
const listBlobs = async (ssbClient) => {
if (!ssbClient.blobs || typeof ssbClient.blobs.ls !== 'function') return new Map();
return new Promise((resolve) => {
pull(ssbClient.blobs.ls({ long: true }), pull.collect((err, arr) => {
if (err) return resolve(new Map());
const out = new Map();
for (const b of arr || []) {
if (!b) continue;
if (typeof b === 'string') out.set(b, 0);
else if (b.id) out.set(b.id, Number(b.size) || 0);
}
resolve(out);
}));
});
};
const readBlob = (ssbClient, id) => new Promise((resolve) => {
pull(ssbClient.blobs.get(id), pull.collect((e, chunks) => {
if (e || !chunks || !chunks.length) return resolve(null);
resolve(Buffer.concat(chunks.map(c => Buffer.isBuffer(c) ? c : Buffer.from(c))));
}));
});
const normalizeOptions = (opts = {}) => {
const scope = String(opts.scope || 'all').toLowerCase() === 'mine' ? 'mine' : 'all';
const withBlobs = opts.blobs === undefined ? true : !(opts.blobs === false || String(opts.blobs) === '0' || String(opts.blobs) === 'false' || String(opts.blobs) === 'off');
const rawMods = Array.isArray(opts.modules) ? opts.modules : (opts.modules ? String(opts.modules).split(',') : []);
const modules = rawMods.map(m => String(m || '').trim().toLowerCase()).filter(m => MODULE_KEYS.includes(m));
const sinceTs = opts.since ? (Date.parse(opts.since) || 0) : 0;
return { scope, withBlobs, modules, sinceTs };
};
const selectMessages = async (ssbClient, options) => {
const me = ssbClient.id;
const all = await readLog(ssbClient);
const picked = [];
const blobIds = new Set();
for (const m of all) {
if (!m || !m.value) continue;
if (options.scope === 'mine' && m.value.author !== me) continue;
if (options.sinceTs && Number(m.value.timestamp || m.timestamp || 0) < options.sinceTs) continue;
if (options.modules.length && !options.modules.includes(moduleOf(m.value.content))) continue;
picked.push(m);
if (options.withBlobs) for (const ref of blobRefsOf(m.value.content)) blobIds.add(ref);
}
if (options.withBlobs && options.scope !== 'mine' && !options.sinceTs && !options.modules.length) {
const sizes = await listBlobs(ssbClient);
for (const id of sizes.keys()) blobIds.add(id);
}
return { picked, blobIds };
};
return {
MODULE_KEYS,
REMINDER_OPTIONS,
moduleOf,
encryptBuffer,
decryptBuffer,
async exportKeys(password) {
const pw = normalizePassword(password);
if (pw.length < 32) throw new Error('Password too short');
const secretPath = secretPathOf();
if (!fs.existsSync(secretPath)) throw new Error(".ssb/secret file doesn't exist");
const original = fs.readFileSync(secretPath);
const encrypted = encryptBuffer(original, pw);
if (!decryptBuffer(encrypted, pw).equals(original)) throw new Error('Backup verification failed; nothing was exported.');
return { filename: 'oasis.enc', data: encrypted };
},
async importKeys({ filePath, password }) {
const pw = normalizePassword(password);
if (!fs.existsSync(filePath)) throw new Error('Encrypted file not found.');
let decrypted;
try { decrypted = decryptBuffer(fs.readFileSync(filePath), pw); } catch (_) { throw new Error('Wrong password or corrupt backup file.'); }
let keys = null;
try { keys = JSON.parse(decrypted.toString('utf8').split('\n').filter(l => !l.trim().startsWith('#')).join('\n')); } catch (_) { keys = null; }
const ssbKeys = require('../server/node_modules/ssb-keys');
const probe = Buffer.from('oasis-keys-check');
const valid = (() => {
try {
if (!keys || keys.curve !== 'ed25519' || typeof keys.public !== 'string' || typeof keys.private !== 'string' || keys.id !== '@' + keys.public) return false;
return ssbKeys.verify(keys, ssbKeys.sign(keys, probe), probe) === true;
} catch (_) { return false; }
})();
if (!valid) throw new Error('The file does not contain a valid Oasis identity.');
const ssbDir = ssbDirOf();
fs.mkdirSync(ssbDir, { recursive: true });
const secretPath = path.join(ssbDir, 'secret');
if (fs.existsSync(secretPath)) {
const bakPath = path.join(ssbDir, 'secret.bak-' + new Date().toISOString().replace(/[:.]/g, '-'));
fs.copyFileSync(secretPath, bakPath);
try { fs.chmodSync(bakPath, 0o600); } catch (_) {}
}
const tmpPath = path.join(ssbDir, 'secret.tmp-' + process.pid);
fs.writeFileSync(tmpPath, decrypted, { mode: 0o600 });
fs.renameSync(tmpPath, secretPath);
try { fs.unlinkSync(filePath); } catch (_) {}
let id = '';
try { id = JSON.parse(decrypted.toString('utf8').split('\n').filter(l => !l.trim().startsWith('#')).join('\n')).id || ''; } catch (_) {}
return { path: secretPath, id };
},
recoveryKit() {
const secretPath = secretPathOf();
if (!fs.existsSync(secretPath)) throw new Error(".ssb/secret file doesn't exist");
const raw = fs.readFileSync(secretPath, 'utf8');
const clean = raw.split('\n').filter(l => !l.trim().startsWith('#')).join('\n').trim();
let id = '';
try { id = JSON.parse(clean).id || ''; } catch (_) {}
return { secret: clean, id, createdAt: new Date().toISOString() };
},
async estimate(opts = {}) {
const ssbClient = await openSsb();
const options = normalizeOptions(opts);
const { picked, blobIds } = await selectMessages(ssbClient, options);
let bytes = 0;
for (const m of picked) bytes += Buffer.byteLength(JSON.stringify({ key: m.key, value: m.value, timestamp: m.timestamp }), 'utf8') + 5;
let blobBytes = 0;
let blobsFound = 0;
if (options.withBlobs && blobIds.size) {
const sizes = await listBlobs(ssbClient);
for (const id of blobIds) {
if (!sizes.has(id)) continue;
blobsFound += 1;
blobBytes += sizes.get(id) || 0;
}
}
return { options, messages: picked.length, blobs: blobsFound, blobsReferenced: blobIds.size, messageBytes: bytes, blobBytes, bytes: bytes + blobBytes };
},
async createBackup(opts = {}, password, outPath) {
const pw = normalizePassword(password);
if (pw.length < 32) throw new Error('Password too short');
const ssbClient = await openSsb();
const options = normalizeOptions(opts);
const { picked, blobIds } = await selectMessages(ssbClient, options);
const salt = crypto.randomBytes(SALT_LEN);
const iv = crypto.randomBytes(IV_LEN);
const cipher = crypto.createCipheriv('aes-256-gcm', deriveKey(pw, salt), iv);
const gzip = zlib.createGzip();
const out = fs.createWriteStream(outPath);
const done = new Promise((resolve, reject) => pipeline(gzip, cipher, out, (err) => err ? reject(err) : resolve()));
out.write(Buffer.concat([BACKUP_MAGIC, salt, iv]));
const meta = { version: 1, createdAt: new Date().toISOString(), author: ssbClient.id, options, messages: picked.length };
await writeChunk(gzip, record(REC_META, Buffer.from(JSON.stringify(meta), 'utf8')));
for (const m of picked) {
await writeChunk(gzip, record(REC_MSG, Buffer.from(JSON.stringify({ key: m.key, value: m.value, timestamp: m.timestamp }), 'utf8')));
}
let files = 0;
const root = stateRoot();
if (root) {
for (const f of listStateFiles(root)) {
let data = null;
try { data = fs.readFileSync(f.full); } catch (_) { continue; }
const head = Buffer.from(JSON.stringify({ rel: f.rel.split(path.sep).join('/'), size: data.length }), 'utf8');
const lenBuf = Buffer.alloc(4);
lenBuf.writeUInt32BE(head.length, 0);
await writeChunk(gzip, record(REC_FILE, Buffer.concat([lenBuf, head, data])));
files += 1;
}
}
let blobs = 0;
if (options.withBlobs) {
for (const id of blobIds) {
const data = await readBlob(ssbClient, id);
if (!data) continue;
const head = Buffer.from(JSON.stringify({ id, size: data.length }), 'utf8');
const lenBuf = Buffer.alloc(4);
lenBuf.writeUInt32BE(head.length, 0);
await writeChunk(gzip, record(REC_BLOB, Buffer.concat([lenBuf, head, data])));
blobs += 1;
}
}
gzip.end();
await done;
fs.appendFileSync(outPath, cipher.getAuthTag());
markBackup();
return { path: outPath, messages: picked.length, blobs, bytes: fs.statSync(outPath).size, filename: `oasis-backup-${new Date().toISOString().slice(0, 10)}.oasisbk` };
},
async createSnapshot(outPath, { sinceMs = 0, now = Date.now() } = {}) {
const ssbClient = await openSsb();
const all = await readLog(ssbClient);
const byAuthor = new Map();
for (const m of all) {
if (!m || !m.value || !m.value.author || m.value.content === undefined || m.value.content === null) continue;
const list = byAuthor.get(m.value.author) || [];
list.push(m);
byAuthor.set(m.value.author, list);
}
const latest = (list) => list.reduce((t, m) => Math.max(t, Number(m.timestamp) || 0, Number(m.value.timestamp) || 0), 0);
const picked = [...byAuthor.values()]
.filter(list => !(sinceMs > 0) || latest(list) >= now - sinceMs)
.sort((a, b) => latest(b) - latest(a))
.flatMap(list => list.sort((a, b) => (a.value.sequence || 0) - (b.value.sequence || 0)));
const gzip = zlib.createGzip();
const out = fs.createWriteStream(outPath);
const done = new Promise((resolve, reject) => pipeline(gzip, out, (err) => err ? reject(err) : resolve()));
out.write(SNAPSHOT_MAGIC);
const meta = { version: 1, kind: 'snapshot', createdAt: new Date().toISOString(), author: ssbClient.id, messages: picked.length, feeds: new Set(picked.map(m => m.value.author)).size, boxed: picked.filter(m => typeof m.value.content === 'string').length, sinceMs: sinceMs > 0 ? sinceMs : null };
await writeChunk(gzip, record(REC_META, Buffer.from(JSON.stringify(meta), 'utf8')));
for (const m of picked) {
await writeChunk(gzip, record(REC_MSG, Buffer.from(JSON.stringify({ key: m.key, value: m.value, timestamp: m.timestamp }), 'utf8')));
}
gzip.end();
await done;
return { path: outPath, messages: picked.length, feeds: meta.feeds, bytes: fs.statSync(outPath).size, createdAt: meta.createdAt };
},
isSnapshotFile(filePath) {
try {
const fd = fs.openSync(filePath, 'r');
const head = Buffer.alloc(SNAPSHOT_MAGIC.length);
fs.readSync(fd, head, 0, head.length, 0);
fs.closeSync(fd);
return head.equals(SNAPSHOT_MAGIC);
} catch (_) { return false; }
},
async fetchSnapshot({ address, connect = null, tier = 'full' }, outPath) {
const ssbClient = await openSsb();
const doConnect = connect || ((addr, cb) => ssbClient.conn.connect(addr, cb));
const discard = () => { try { fs.unlinkSync(outPath); } catch (_) {} return null; };
try {
const rpc = await new Promise((resolve, reject) => doConnect(address, (err, r) => err ? reject(err) : resolve(r)));
if (!rpc || !rpc.snapshot || typeof rpc.snapshot.get !== 'function') return null;
const info = await new Promise((resolve) => rpc.snapshot.info({ tier }, (err, i) => resolve(err ? null : i)));
if (!info || !(info.bytes > 0) || info.bytes > SNAPSHOT_MAX_BYTES) return null;
const out = fs.createWriteStream(outPath);
let written = 0;
await new Promise((resolve, reject) => {
pull(
rpc.snapshot.get({ tier }),
pull.map(c => Buffer.isBuffer(c) ? c : Buffer.from(c)),
pull.through(c => { written += c.length; }),
toPull.sink(out, (err) => err ? reject(err) : resolve())
);
});
if (written > SNAPSHOT_MAX_BYTES || !this.isSnapshotFile(outPath)) return discard();
return { address, tier, path: outPath, bytes: written, createdAt: info.createdAt };
} catch (_) { return discard(); }
},
startBootstrap({ address, connect = null, tiers = ['recent', 'full'] }) {
if (restoreJob && restoreJob.running) return restoreJob;
const empty = () => ({ messages: 0, skipped: 0, failed: 0, forked: 0, blobs: 0, blobsSkipped: 0, percent: 0 });
const job = { running: true, source: 'snapshot', phase: null, tier: null, tiers: [], address, startedAt: new Date().toISOString(), finishedAt: null, progress: empty(), result: null, error: null };
restoreJob = job;
const total = { ...empty(), files: 0, forks: [], errors: [], meta: null };
const run = async () => {
for (const tier of tiers) {
const tmp = path.join(os.tmpdir(), `oasis-snapshot-${tier}-${Date.now()}-${Math.random().toString(36).slice(2)}.oasissn`);
job.tier = tier;
job.phase = 'download';
try {
const got = await this.fetchSnapshot({ address, connect, tier }, tmp);
if (!got) continue;
job.phase = 'restore';
const res = await this.restoreBackup({ filePath: tmp, password: '', stateFiles: false, onProgress: (p) => { job.progress = { ...total, ...p, messages: total.messages + p.messages, skipped: total.skipped + p.skipped, failed: total.failed + p.failed, forked: total.forked + p.forked }; } });
for (const k of ['messages', 'skipped', 'failed', 'forked', 'blobs', 'blobsSkipped', 'files']) total[k] += Number(res[k]) || 0;
total.forks.push(...(res.forks || []));
total.errors.push(...(res.errors || []));
total.meta = res.meta || total.meta;
job.tiers.push({ tier, messages: res.messages, skipped: res.skipped, bytes: got.bytes });
} finally { try { fs.unlinkSync(tmp); } catch (_) {} }
}
if (!job.tiers.length) throw new Error('snapshot not available');
total.percent = 100;
return total;
};
job.promise = run()
.then((res) => { job.result = res; job.progress = { ...job.progress, ...res }; return res; })
.catch((err) => { job.error = err && err.message ? err.message : String(err); return null; })
.finally(() => { job.running = false; job.phase = null; job.finishedAt = new Date().toISOString(); });
return job;
},
async readBackup(filePath, password, onRecord) {
const size = fs.statSync(filePath).size;
const snapshot = this.isSnapshotFile(filePath);
const pw = snapshot ? '' : normalizePassword(password);
const headLen = snapshot ? SNAPSHOT_MAGIC.length : BACKUP_MAGIC.length + SALT_LEN + IV_LEN;
const tailLen = snapshot ? 0 : TAG_LEN;
if (size < headLen + tailLen) throw new Error('Unrecognized or corrupt backup file.');
const fd = fs.openSync(filePath, 'r');
const head = Buffer.alloc(headLen);
fs.readSync(fd, head, 0, headLen, 0);
const tag = Buffer.alloc(TAG_LEN);
if (!snapshot) fs.readSync(fd, tag, 0, TAG_LEN, size - TAG_LEN);
fs.closeSync(fd);
if (!snapshot && !head.slice(0, BACKUP_MAGIC.length).equals(BACKUP_MAGIC)) throw new Error('Unrecognized or corrupt backup file.');
let decipher = null;
if (!snapshot) {
const salt = head.slice(BACKUP_MAGIC.length, BACKUP_MAGIC.length + SALT_LEN);
const iv = head.slice(BACKUP_MAGIC.length + SALT_LEN, headLen);
decipher = crypto.createDecipheriv('aes-256-gcm', deriveKey(pw, salt), iv);
decipher.setAuthTag(tag);
}
const gunzip = zlib.createGunzip();
const total = size - headLen - tailLen;
const source = fs.createReadStream(filePath, { start: headLen, end: size - tailLen - 1 });
let streamError = null;
if (decipher) pipeline(source, decipher, gunzip, (err) => { if (err) streamError = err; });
else pipeline(source, gunzip, (err) => { if (err) streamError = err; });
const chunks = [];
let pendingLen = 0;
let meta = null;
const compact = () => {
if (chunks.length < 2) return;
const all = Buffer.concat(chunks, pendingLen);
chunks.length = 0;
chunks.push(all);
};
const nextRecord = () => {
if (pendingLen < 5) return null;
if (chunks[0].length < 5) compact();
const type = chunks[0].readUInt8(0);
const len = chunks[0].readUInt32BE(1);
if (len > REC_MAX_LEN) throw new Error('Backup record too large.');
if (pendingLen < 5 + len) return null;
compact();
const all = chunks[0];
chunks.length = 0;
pendingLen = all.length - 5 - len;
if (pendingLen) chunks.push(all.slice(5 + len));
return { type, payload: all.slice(5, 5 + len) };
};
const handle = async (type, payload) => {
const read = Math.min(total, source.bytesRead || 0);
if (type === REC_META) { meta = JSON.parse(payload.toString('utf8')); await onRecord({ type: 'meta', meta, read, total }); }
else if (type === REC_MSG) await onRecord({ type: 'msg', msg: JSON.parse(payload.toString('utf8')), read, total });
else if (type === REC_BLOB) {
const hl = payload.readUInt32BE(0);
const header = JSON.parse(payload.slice(4, 4 + hl).toString('utf8'));
await onRecord({ type: 'blob', id: header.id, data: payload.slice(4 + hl), read, total });
}
else if (type === REC_FILE) {
const hl = payload.readUInt32BE(0);
const header = JSON.parse(payload.slice(4, 4 + hl).toString('utf8'));
await onRecord({ type: 'file', rel: header.rel, data: payload.slice(4 + hl), read, total });
}
};
try {
for await (const chunk of gunzip) {
chunks.push(chunk);
pendingLen += chunk.length;
let rec;
while ((rec = nextRecord())) await handle(rec.type, rec.payload);
}
} catch (e) {
source.destroy();
if (streamError || /unsupported state|auth|bad decrypt|incorrect header|invalid/i.test(String(e && e.message))) throw new Error('Wrong password or corrupt backup file.');
throw e;
}
if (streamError) throw new Error('Wrong password or corrupt backup file.');
return meta;
},
async restoreBackup({ filePath, password, onProgress, stateFiles = true }) {
const ssbClient = await openSsb();
let validate = null;
try { validate = require('../server/node_modules/ssb-validate'); } catch (_) {}
const seqKey = (v) => `${v.author}::${v.sequence}`;
const existing = new Map();
for (const m of await readLog(ssbClient)) if (m && m.value) existing.set(seqKey(m.value), m.key);
const stats = { messages: 0, skipped: 0, failed: 0, forked: 0, blobs: 0, blobsSkipped: 0, files: 0, percent: 0, forks: [], errors: [], meta: null };
const forkAuthors = new Map();
const errorCounts = new Map();
const noteError = (err) => {
const reason = String(err && err.message ? err.message : err || 'unknown error').replace(/, on feed:@[^\s]+/g, '').replace(/in state:\{.*?\}/g, '').trim();
errorCounts.set(reason, (errorCounts.get(reason) || 0) + 1);
};
const addMsg = (value) => new Promise((resolve) => {
if (typeof ssbClient.add !== 'function') return resolve(new Error('add is not available'));
ssbClient.add(value, (err) => resolve(err || null));
});
const addBlob = (data) => new Promise((resolve) => {
pull(pull.values([data]), ssbClient.blobs.add((err, ref) => resolve(err ? null : ref)));
});
const getByKey = (key) => new Promise((resolve) => {
if (!key || typeof ssbClient.get !== 'function') return resolve(false);
try { ssbClient.get(key, (err, value) => resolve(!err && !!value)); } catch (_) { resolve(false); }
});
const storedByKey = async (msg) => {
if (await getByKey(msg.key)) return true;
let computed = null;
try { computed = validate ? validate.id(msg.value) : null; } catch (_) { computed = null; }
return computed && computed !== msg.key ? getByKey(computed) : false;
};
const supportsLs = !!(ssbClient.blobs && typeof ssbClient.blobs.ls === 'function');
const present = supportsLs ? await listBlobs(ssbClient) : new Map();
const hasBlob = (id) => supportsLs ? Promise.resolve(present.has(id)) : new Promise((resolve) => ssbClient.blobs.has(id, (err, has) => resolve(!err && !!has)));
const deferred = [];
const report = () => { if (typeof onProgress === 'function') onProgress({ ...stats }); };
const sameMessage = (storedKey, msg) => {
if (storedKey === msg.key) return true;
try { return storedKey === validate.id(msg.value); } catch (_) { return false; }
};
const tryAdd = async (msg) => {
const k = seqKey(msg.value);
if (existing.has(k)) {
if (sameMessage(existing.get(k), msg)) { stats.skipped += 1; return true; }
stats.forked += 1;
forkAuthors.set(msg.value.author, (forkAuthors.get(msg.value.author) || 0) + 1);
return true;
}
const err = await addMsg(msg.value);
if (err) {
if (!(await storedByKey(msg))) return err;
stats.skipped += 1;
existing.set(k, msg.key);
return true;
}
stats.messages += 1;
existing.set(k, msg.key);
return true;
};
const meta = await this.readBackup(filePath, password, async (rec) => {
if (rec.total) stats.percent = Math.min(100, (rec.read / rec.total) * 100);
if (rec.type === 'meta') { stats.meta = rec.meta; return; }
if (rec.type === 'msg') {
if (!rec.msg || !rec.msg.value || !rec.msg.value.author) { stats.failed += 1; noteError('malformed message'); report(); return; }
const res = await tryAdd(rec.msg);
if (res !== true) deferred.push({ msg: rec.msg, err: res });
report();
return;
}
if (rec.type === 'file') {
if (!stateFiles) { stats.skipped += 1; report(); return; }
const root = stateRoot();
const rel = typeof rec.rel === 'string' ? rec.rel.replace(/\\/g, '/') : '';
const parts = rel.split('/').filter(Boolean);
const safe = !!root && parts.length > 0 && !NOT_BACKED_UP.has(parts[0]) && !path.isAbsolute(rel) && parts.every(p => p !== '.' && p !== '..' && !/[\0]/.test(p));
const rootAbs = root ? path.resolve(root) : '';
const target = safe ? path.resolve(rootAbs, ...parts) : '';
if (!safe || !target.startsWith(rootAbs + path.sep)) { stats.failed += 1; noteError('state file rejected'); report(); return; }
try {
fs.mkdirSync(path.dirname(target), { recursive: true });
if (fs.existsSync(target)) { try { fs.copyFileSync(target, `${target}.before-restore`); } catch (_) {} }
fs.writeFileSync(target, rec.data);
stats.files += 1;
} catch (_) { stats.failed += 1; noteError('state file could not be written'); }
report();
return;
}
if (rec.type === 'blob') {
if (await hasBlob(rec.id)) { stats.blobsSkipped += 1; report(); return; }
const ref = await addBlob(rec.data);
if (ref) { stats.blobs += 1; present.set(ref, rec.data.length); } else { stats.failed += 1; noteError('blob could not be stored'); }
report();
}
});
deferred.sort((a, b) => a.msg.value.author === b.msg.value.author ? (a.msg.value.sequence || 0) - (b.msg.value.sequence || 0) : String(a.msg.value.author).localeCompare(String(b.msg.value.author)));
let pending = deferred;
while (pending.length) {
const next = [];
for (const item of pending) {
const res = await tryAdd(item.msg);
if (res !== true) next.push({ msg: item.msg, err: res });
}
report();
if (next.length === pending.length) { pending = next; break; }
pending = next;
}
for (const item of pending) { stats.failed += 1; noteError(item.err); }
stats.percent = 100;
stats.forks = Array.from(forkAuthors.entries()).map(([author, count]) => ({ author, count, mine: author === ssbClient.id }));
stats.errors = Array.from(errorCounts.entries()).sort((a, b) => b[1] - a[1]).slice(0, 5).map(([reason, count]) => ({ reason, count }));
try { fs.unlinkSync(filePath); } catch (_) {}
report();
return { ...stats, meta: stats.meta || meta };
},
startRestore({ filePath, password, source = 'file' }) {
if (restoreJob && restoreJob.running) return restoreJob;
const job = { running: true, source, startedAt: new Date().toISOString(), finishedAt: null, progress: { messages: 0, skipped: 0, failed: 0, forked: 0, blobs: 0, blobsSkipped: 0, percent: 0 }, result: null, error: null };
restoreJob = job;
job.promise = this.restoreBackup({ filePath, password, onProgress: (p) => { job.progress = p; } })
.then((res) => { job.result = res; return res; })
.catch((err) => { job.error = err && err.message ? err.message : String(err); try { fs.unlinkSync(filePath); } catch (_) {} return null; })
.finally(() => { job.running = false; job.finishedAt = new Date().toISOString(); });
return job;
},
restoreStatus() {
if (!restoreJob) return null;
const { promise, ...rest } = restoreJob;
return rest;
},
reminderState() { return reminderState(); },
setReminderDays(days) { return setReminderDays(days); },
markBackup() { return markBackup(); },
async rebuildIndexes() {
const started = Date.now();
const ssbClient = await openSsb();
const indexDir = path.join(require('../configs/state-manager').ssbDir(), 'db2', 'indexes');
const measure = (dir) => {
let files = 0, bytes = 0;
const walk = (d) => {
let entries = [];
try { entries = fs.readdirSync(d, { withFileTypes: true }); } catch (_) { return; }
for (const entry of entries) {
const full = path.join(d, entry.name);
if (entry.isDirectory()) { walk(full); continue; }
try { bytes += fs.statSync(full).size; files += 1; } catch (_) {}
}
};
walk(dir);
return { files, bytes };
};
const before = measure(indexDir);
let error = null;
try {
await new Promise((resolve, reject) => {
try { ssbClient.rebuild((err) => err ? reject(err) : resolve()); } catch (e) { reject(e); }
});
} catch (e) { error = e && e.message ? e.message : String(e); }
const after = measure(indexDir);
let totalMessages = 0;
try { totalMessages = (await readLog(ssbClient)).length; } catch (_) {}
return {
checkedAt: new Date().toISOString(),
tookMs: Date.now() - started,
ok: !error,
error,
totalMessages,
indexes: { path: indexDir, files: after.files, bytes: after.bytes, deltaBytes: after.bytes - before.bytes }
};
},
async verify({ author = null } = {}) {
const started = Date.now();
const ssbClient = await openSsb();
const me = author || ssbClient.id;
const all = await readLog(ssbClient);
let validate = null;
try { validate = require('../server/node_modules/ssb-validate'); } catch (_) {}
const own = all.filter(m => m && m.value && m.value.author === me).sort((a, b) => (a.value.sequence || 0) - (b.value.sequence || 0));
const feed = { messages: own.length, lastSequence: own.length ? own[own.length - 1].value.sequence : 0, gaps: [], brokenLinks: [], badHashes: [], badSignatures: [], duplicates: [] };
let expected = 1;
let prevKey = null;
const seen = new Set();
for (const m of own) {
const seq = Number(m.value.sequence) || 0;
if (seen.has(seq)) feed.duplicates.push(seq);
seen.add(seq);
if (seq !== expected) { feed.gaps.push({ expected, found: seq }); expected = seq; }
if (prevKey && m.value.previous !== prevKey) feed.brokenLinks.push(seq);
if (validate) {
try { if (validate.id(m.value) !== m.key) feed.badHashes.push(seq); } catch (_) { feed.badHashes.push(seq); }
try { if (validate.checkInvalidOOO(m.value, null)) feed.badSignatures.push(seq); } catch (_) { feed.badSignatures.push(seq); }
}
prevKey = m.key;
expected = seq + 1;
}
feed.ok = !feed.gaps.length && !feed.brokenLinks.length && !feed.badHashes.length && !feed.badSignatures.length && !feed.duplicates.length;
const bySeq = new Map();
for (const m of all) {
if (!m || !m.value || !m.value.author) continue;
const k = `${m.value.author}::${m.value.sequence}`;
if (!bySeq.has(k)) bySeq.set(k, new Set());
bySeq.get(k).add(m.key);
}
const forkMap = new Map();
for (const [k, keys] of bySeq) {
if (keys.size < 2) continue;
const [feedId, seq] = k.split('::');
if (!forkMap.has(feedId)) forkMap.set(feedId, []);
forkMap.get(feedId).push({ sequence: Number(seq), keys: Array.from(keys) });
}
const forks = Array.from(forkMap.entries()).map(([feedId, points]) => ({ author: feedId, mine: feedId === me, points: points.sort((a, b) => a.sequence - b.sequence) }));
const referenced = new Set();
const referencedOwn = new Set();
for (const m of all) {
if (!m || !m.value) continue;
for (const ref of blobRefsOf(m.value.content)) {
referenced.add(ref);
if (m.value.author === me) referencedOwn.add(ref);
}
}
const present = await listBlobs(ssbClient);
const orphan = [];
let orphanBytes = 0;
for (const [id, size] of present) if (!referenced.has(id)) { orphan.push(id); orphanBytes += size || 0; }
const missing = [];
const missingOwn = [];
for (const id of referenced) {
if (!present.size || present.has(id)) continue;
missing.push(id);
if (referencedOwn.has(id)) missingOwn.push(id);
}
return {
author: me, checkedAt: new Date().toISOString(), tookMs: Date.now() - started, totalMessages: all.length,
feed, forks, forkCount: forks.length,
blobs: { present: present.size, referenced: referenced.size, orphan: orphan.length, orphanBytes, missing: missing.length, missingOwn: missingOwn.length, orphanIds: orphan.slice(0, 50), missingIds: missing.slice(0, 50), missingOwnIds: missingOwn.slice(0, 50), listed: present.size > 0 || referenced.size === 0 }
};
}
};
};
Object.assign(module.exports, { reminderState, setReminderDays, markBackup, REMINDER_OPTIONS, MODULE_KEYS });

1732
src/models/banking_model.js
View file

File diff suppressed because it is too large Load diff

161
src/models/blobcache_model.js Normal file
View file

@ -0,0 +1,161 @@
const fs = require('fs');
const pull = require('../server/node_modules/pull-stream');
const BLOB_RE = /&[A-Za-z0-9+/=]{44}\.sha256/g;
const DEFAULT_MAX_MB = 0;
const PROTECT_MS = 24 * 60 * 60 * 1000;
const FLUSH_MS = 60 * 1000;
const pinsPath = () => {
try { return require('../configs/state-manager').statePath('blob-pins.json'); } catch (_) { return null; }
};
const accessPath = () => {
try { return require('../configs/state-manager').statePath('blob-access.json'); } catch (_) { return null; }
};
module.exports = ({ cooler } = {}) => {
const openSsb = async () => cooler.open();
let access = null;
let dirty = false;
let flushTimer = null;
const loadAccess = () => {
if (access) return access;
access = {};
const p = accessPath();
if (p) { try { access = JSON.parse(fs.readFileSync(p, 'utf8')) || {}; } catch (_) { access = {}; } }
return access;
};
const flush = () => {
if (!dirty) return;
const p = accessPath();
if (!p) return;
try { fs.writeFileSync(p, JSON.stringify(loadAccess())); dirty = false; } catch (_) {}
};
const normalizeId = (raw) => {
let id = String(raw || '').trim();
try { id = decodeURIComponent(id); } catch (_) {}
if (!id) return null;
if (!id.startsWith('&')) id = `&${id}`;
return /^&[A-Za-z0-9+/=]{44}\.sha256$/.test(id) ? id : null;
};
const touch = (raw) => {
const id = normalizeId(raw);
if (!id) return;
loadAccess()[id] = Date.now();
dirty = true;
if (!flushTimer) {
flushTimer = setTimeout(() => { flushTimer = null; flush(); }, FLUSH_MS);
if (typeof flushTimer.unref === 'function') flushTimer.unref();
}
};
const listBlobs = async (ssb) => new Promise((resolve) => {
if (!ssb.blobs || typeof ssb.blobs.ls !== 'function') return resolve([]);
pull(ssb.blobs.ls({ long: true }), pull.collect((err, arr) => {
if (err) return resolve([]);
resolve((arr || []).filter(b => b && b.id).map(b => ({ id: b.id, size: Number(b.size) || 0, ts: Number(b.ts) || 0 })));
}));
});
const loadPins = () => {
const p = pinsPath();
if (!p) return [];
try { const arr = JSON.parse(fs.readFileSync(p, 'utf8')); return Array.isArray(arr) ? arr : []; } catch (_) { return []; }
};
const pin = (ids) => {
const p = pinsPath();
if (!p) return;
const set = new Set(loadPins());
for (const raw of (Array.isArray(ids) ? ids : [ids])) { const id = normalizeId(raw); if (id) set.add(id); }
try { fs.writeFileSync(p, JSON.stringify(Array.from(set))); } catch (_) {}
};
const getContent = (ssb, key) => new Promise((resolve) => {
try { ssb.get(key, (err, v) => resolve(!err && v && v.content && typeof v.content === 'object' ? v.content : null)); } catch (_) { resolve(null); }
});
const ownBlobIds = async (ssb) => {
const own = new Set();
const msgs = await new Promise((resolve) => {
pull(ssb.createUserStream({ id: ssb.id, reverse: false }), pull.collect((err, arr) => resolve(err ? [] : (arr || []))));
});
const spreads = new Map();
const tombstoned = new Set();
for (const m of msgs) {
const content = m && m.value && m.value.content;
if (!content || typeof content !== 'object') continue;
if (content.type === 'spread' && typeof content.link === 'string') spreads.set(m.key, content.link);
else if (content.type === 'tombstone' && typeof content.target === 'string') tombstoned.add(content.target);
for (const id of (JSON.stringify(content).match(BLOB_RE) || [])) own.add(id);
}
if (ssb.private && typeof ssb.private.read === 'function') {
const boxed = await new Promise((resolve) => {
pull(ssb.private.read({ reverse: false }), pull.collect((err, arr) => resolve(err ? [] : (arr || []))));
});
for (const m of boxed) {
const content = m && m.value && m.value.content;
if (content && typeof content === 'object') for (const id of (JSON.stringify(content).match(BLOB_RE) || [])) own.add(id);
}
}
for (const id of loadPins()) own.add(id);
for (const [key, link] of spreads) {
if (tombstoned.has(key)) continue;
const seeded = await getContent(ssb, link);
if (!seeded || (seeded.type !== 'torrent' && seeded.type !== 'file')) continue;
for (const id of (JSON.stringify(seeded).match(BLOB_RE) || [])) own.add(id);
}
return own;
};
const usage = async () => {
const ssb = await openSsb();
const blobs = await listBlobs(ssb);
return { count: blobs.length, bytes: blobs.reduce((n, b) => n + b.size, 0) };
};
const collect = async ({ maxBytes, now = Date.now(), protectMs = PROTECT_MS } = {}) => {
const limit = Number(maxBytes);
const ssb = await openSsb();
const blobs = await listBlobs(ssb);
const total = blobs.reduce((n, b) => n + b.size, 0);
const result = { before: total, after: total, deleted: 0, freed: 0, kept: blobs.length };
if (!(limit > 0) || total <= limit) return result;
const own = await ownBlobIds(ssb);
const seen = loadAccess();
const candidates = blobs
.filter(b => !own.has(b.id))
.map(b => ({ ...b, last: Math.max(Number(seen[b.id]) || 0, b.ts) }))
.filter(b => now - b.last > protectMs)
.sort((a, b) => a.last - b.last);
let current = total;
for (const b of candidates) {
if (current <= limit) break;
const removed = await new Promise((resolve) => ssb.blobs.rm(b.id, (err) => resolve(!err)));
if (!removed) continue;
current -= b.size;
result.deleted += 1;
result.freed += b.size;
delete seen[b.id];
dirty = true;
}
result.after = current;
result.kept = blobs.length - result.deleted;
flush();
return result;
};
const maxBytesFor = (cfg, { isPublic = false } = {}) => {
const mb = Number(cfg && cfg.blobCache && (isPublic ? cfg.blobCache.pubMaxMB : cfg.blobCache.maxMB));
if (!Number.isFinite(mb) || mb <= 0) return 0;
return Math.round(mb * 1024 * 1024);
};
return { touch, usage, collect, ownBlobIds, maxBytesFor, flush, pin, DEFAULT_MAX_MB };
};
module.exports.DEFAULT_MAX_MB = DEFAULT_MAX_MB;

256
src/models/blockchain_model.js
View file

@ -2,6 +2,8 @@ const pull = require('../server/node_modules/pull-stream');
const config = require('../server/ssb_config');
const { getConfig } = require('../configs/config-manager.js');
const { buildValidatedTombstoneSet } = require('./tombstone_validator');
const { readContentWindow, requestScope, logTip } = require('./typed_log');
const { versionGroups, versionEntry } = require('../backend/dedupe');
const logLimit = getConfig().ssbLogStream?.limit || 1000;
module.exports = ({ cooler, tribeCrypto, tribesModel }) => {
@ -11,6 +13,7 @@ module.exports = ({ cooler, tribeCrypto, tribesModel }) => {
if (!ssb) ssb = await cooler.open();
return ssb;
};
const nameByFeedId = new Map();
const HIDDEN_ENVELOPE_TYPES = new Set([
'tribe-keys-distrib',
@ -19,7 +22,7 @@ module.exports = ({ cooler, tribeCrypto, tribesModel }) => {
]);
const isHiddenBoxedContent = (rawContent) =>
typeof rawContent === 'string' && rawContent.endsWith('.box');
typeof rawContent === 'string' && /\.box\d*$/.test(rawContent);
const buildAccessibleTribeIds = async () => {
const set = new Set();
@ -41,16 +44,10 @@ module.exports = ({ cooler, tribeCrypto, tribesModel }) => {
const hasBlob = async (ssbClient, url) =>
new Promise(resolve => ssbClient.blobs.has(url, (err, has) => resolve(!err && has)));
const isClosedSold = s =>
String(s || '').toUpperCase() === 'SOLD' || String(s || '').toUpperCase() === 'CLOSED';
const projectRank = (status) => {
const S = String(status || '').toUpperCase();
if (S === 'COMPLETED') return 3;
if (S === 'ACTIVE') return 2;
if (S === 'PAUSED') return 1;
if (S === 'CANCELLED') return 0;
return -1;
const isReplacedBlock = (vg, tombstoned, block) => {
const c = block.content;
const rootDeleted = !c.replaces && c?.type === 'forum' && c.root && tombstoned.has(c.root);
return vg.tipOf(block.id) !== block.id || vg.isForeignCopy(block.id) || tombstoned.has(block.id) || !!rootDeleted;
};
const safeDecode = (s) => {
@ -127,115 +124,83 @@ module.exports = ({ cooler, tribeCrypto, tribesModel }) => {
return nameByFeedId;
};
let windowCache = null;
const readWindow = async (ssbClient) => {
const tip = await logTip(ssbClient).catch(() => null);
if (tip && windowCache && windowCache.tip === tip) return windowCache.msgs;
const msgs = await readContentWindow(ssbClient, logLimit);
if (tip) windowCache = { tip, msgs };
return msgs;
};
async function buildBlocks(showLogs, me) {
const ssbClient = await openSsb();
const results = await readWindow(ssbClient);
const tombstoned = buildValidatedTombstoneSet(results);
const idToBlock = new Map();
const versions = [];
const fpIdx = tribeCrypto ? tribeCrypto.buildFingerprintIndex() : null;
const accessibleTribeIds = await buildAccessibleTribeIds();
for (const msg of results) {
const k = msg.key;
let c = msg.value?.content;
const author = msg.value?.author;
if (isHiddenBoxedContent(c) || msg.value?.private === true) continue;
if (showLogs && typeof c === 'string' && author === me) {
try {
const dec = ssbClient.private.unbox({ key: k, value: msg.value, timestamp: msg.timestamp || msg.value?.timestamp || 0 });
c = dec?.value?.content;
} catch { c = null; }
}
if (!c?.type) continue;
if (HIDDEN_ENVELOPE_TYPES.has(c.type)) continue;
if (tribeCrypto && tribeCrypto.isTribeMsg(c)) {
const r = fpIdx ? tribeCrypto.unwrapMsg(c, fpIdx) : null;
if (!r || !r.body) continue;
const inner = r.body;
const innerType = inner.k === 'tribe' ? 'tribe' : (inner.k === 'tribe-content' ? `tribe-content:${inner.contentType || ''}` : inner.k || 'tribe-msg');
c = { type: innerType, _decrypted: true, _rootId: r.rootId, ...inner };
} else if (c.tribeId && !accessibleTribeIds.has(c.tribeId)) {
continue;
}
if (c.type === 'about') {
const aboutId = String(c.about || author || '').trim();
const nm = typeof c.name === 'string' ? c.name.trim() : '';
if (aboutId && nm && !nameByFeedId.has(aboutId)) nameByFeedId.set(aboutId, nm);
}
if (c.type === 'tombstone' && c.target) {
idToBlock.set(k, { id: k, author, ts: msg.value.timestamp, type: c.type, content: c, size: Buffer.byteLength(JSON.stringify(msg.value), 'utf8') });
continue;
}
versions.push(versionEntry({ key: k, value: { ...msg.value, content: c } }));
idToBlock.set(k, { id: k, author, ts: msg.value.timestamp, type: c.type, content: c, size: Buffer.byteLength(JSON.stringify(msg.value), 'utf8') });
}
const vg = versionGroups(versions);
const blockData = Array.from(idToBlock.values()).map(block => ({
...block,
isTombstoned: tombstoned.has(block.id),
isReplaced: isReplacedBlock(vg, tombstoned, block)
}));
return blockData;
}
return {
async listBlockchain(filter = 'all', userId, search = {}) {
const ssbClient = await openSsb();
const results = await new Promise((resolve, reject) =>
pull(
ssbClient.createLogStream({ reverse: true, limit: logLimit }),
pull.collect((err, msgs) => err ? reject(err) : resolve(msgs))
)
);
const tombstoned = buildValidatedTombstoneSet(results);
const idToBlock = new Map();
const referencedAsReplaces = new Set();
const nameByFeedId = new Map();
const showLogs = (filter === 'logs' || filter === 'LOGS');
const me = userId || config.keys.id;
const fpIdx = tribeCrypto ? tribeCrypto.buildFingerprintIndex() : null;
const accessibleTribeIds = await buildAccessibleTribeIds();
for (const msg of results) {
const k = msg.key;
let c = msg.value?.content;
const author = msg.value?.author;
if (isHiddenBoxedContent(c)) continue;
if (showLogs && typeof c === 'string' && author === me) {
try {
const dec = ssbClient.private.unbox({ key: k, value: msg.value, timestamp: msg.timestamp || msg.value?.timestamp || 0 });
c = dec?.value?.content;
} catch { c = null; }
}
if (!c?.type) continue;
if (HIDDEN_ENVELOPE_TYPES.has(c.type)) continue;
if (tribeCrypto && tribeCrypto.isTribeMsg(c)) {
const r = fpIdx ? tribeCrypto.unwrapMsg(c, fpIdx) : null;
if (!r || !r.body) continue;
const inner = r.body;
const innerType = inner.k === 'tribe' ? 'tribe' : (inner.k === 'tribe-content' ? `tribe-content:${inner.contentType || ''}` : inner.k || 'tribe-msg');
c = { type: innerType, _decrypted: true, _rootId: r.rootId, ...inner };
} else if (c.tribeId && !accessibleTribeIds.has(c.tribeId)) {
continue;
}
if (c.type === 'about') {
const aboutId = String(c.about || author || '').trim();
const nm = typeof c.name === 'string' ? c.name.trim() : '';
if (aboutId && nm && !nameByFeedId.has(aboutId)) nameByFeedId.set(aboutId, nm);
}
if (c.type === 'tombstone' && c.target) {
idToBlock.set(k, { id: k, author, ts: msg.value.timestamp, type: c.type, content: c, size: Buffer.byteLength(JSON.stringify(msg.value), 'utf8') });
continue;
}
if (c.replaces) referencedAsReplaces.add(c.replaces);
idToBlock.set(k, { id: k, author, ts: msg.value.timestamp, type: c.type, content: c, size: Buffer.byteLength(JSON.stringify(msg.value), 'utf8') });
}
const tipBlocks = [];
for (const [id, block] of idToBlock.entries()) {
if (!referencedAsReplaces.has(id) && block.content.replaces) tipBlocks.push(block);
}
for (const [id, block] of idToBlock.entries()) {
if (!block.content.replaces && !referencedAsReplaces.has(id)) tipBlocks.push(block);
}
const groups = {};
for (const block of tipBlocks) {
const ancestor = block.content.replaces || block.id;
if (!groups[ancestor]) groups[ancestor] = [];
groups[ancestor].push(block);
}
const liveTipIds = new Set();
for (const groupBlocks of Object.values(groups)) {
let best = groupBlocks[0];
for (const block of groupBlocks) {
if (block.type === 'market') {
if (isClosedSold(block.content.status) && !isClosedSold(best.content.status)) {
best = block;
} else if ((block.content.status === best.content.status) && block.ts > best.ts) {
best = block;
}
} else if (block.type === 'project') {
const br = projectRank(best.content.status);
const cr = projectRank(block.content.status);
if (cr > br || (cr === br && block.ts > best.ts)) best = block;
} else if (block.type === 'job' || block.type === 'forum') {
if (block.ts > best.ts) best = block;
} else {
if (block.ts > best.ts) best = block;
}
}
liveTipIds.add(best.id);
}
const blockData = Array.from(idToBlock.values()).map(block => {
const c = block.content;
const rootDeleted = c?.type === 'forum' && c.root && tombstoned.has(c.root);
return {
...block,
isTombstoned: tombstoned.has(block.id),
isReplaced: c.replaces
? (!liveTipIds.has(block.id) || tombstoned.has(block.id))
: referencedAsReplaces.has(block.id) || tombstoned.has(block.id) || rootDeleted
};
});
let filtered = blockData;
const store = requestScope.getStore();
const memoKey = `blockchainBase:${showLogs}:${me}`;
if (store && !store[memoKey]) store[memoKey] = buildBlocks(showLogs, me);
const blockData = await (store ? store[memoKey] : buildBlocks(showLogs, me));
let filtered = blockData.slice();
if (filter === 'RECENT' || filter === 'recent') {
const now = Date.now();
@ -267,7 +232,7 @@ module.exports = ({ cooler, tribeCrypto, tribesModel }) => {
const needsNameSearch = !!authorQ && !authorQ.toLowerCase().includes('.ed25519');
if (needsNameSearch) {
aboutIndex = await buildNameIndexFromAbout(ssbClient, 10000);
aboutIndex = await buildNameIndexFromAbout(await openSsb(), 10000);
for (const [fid, nm] of aboutIndex.entries()) {
if (!nameByFeedId.has(fid)) nameByFeedId.set(fid, nm);
}
@ -293,17 +258,12 @@ module.exports = ({ cooler, tribeCrypto, tribesModel }) => {
async getBlockById(id, userId) {
const ssbClient = await openSsb();
const results = await new Promise((resolve, reject) =>
pull(
ssbClient.createLogStream({ reverse: true, limit: logLimit }),
pull.collect((err, msgs) => err ? reject(err) : resolve(msgs))
)
);
const results = await readWindow(ssbClient);
const me = userId || config.keys.id;
const tombstoned = buildValidatedTombstoneSet(results);
const idToBlock = new Map();
const referencedAsReplaces = new Set();
const versions = [];
const fpIdx = tribeCrypto ? tribeCrypto.buildFingerprintIndex() : null;
const accessibleTribeIds = await buildAccessibleTribeIds();
@ -311,7 +271,7 @@ module.exports = ({ cooler, tribeCrypto, tribesModel }) => {
const k = msg.key;
let c = msg.value?.content;
const author = msg.value?.author;
if (isHiddenBoxedContent(c)) continue;
if (isHiddenBoxedContent(c) || msg.value?.private === true) continue;
if (typeof c === 'string' && author === me) {
try {
const dec = ssbClient.private.unbox({ key: k, value: msg.value, timestamp: msg.timestamp || msg.value?.timestamp || 0 });
@ -333,48 +293,10 @@ module.exports = ({ cooler, tribeCrypto, tribesModel }) => {
idToBlock.set(k, { id: k, author, ts: msg.value.timestamp, type: c.type, content: c, size: Buffer.byteLength(JSON.stringify(msg.value), 'utf8') });
continue;
}
if (c.replaces) referencedAsReplaces.add(c.replaces);
versions.push(versionEntry({ key: k, value: { ...msg.value, content: c } }));
idToBlock.set(k, { id: k, author, ts: msg.value.timestamp, type: c.type, content: c, size: Buffer.byteLength(JSON.stringify(msg.value), 'utf8') });
}
const tipBlocks = [];
for (const [bid, block] of idToBlock.entries()) {
if (!referencedAsReplaces.has(bid) && block.content.replaces) tipBlocks.push(block);
}
for (const [bid, block] of idToBlock.entries()) {
if (!block.content.replaces && !referencedAsReplaces.has(bid)) tipBlocks.push(block);
}
const groups = {};
for (const block of tipBlocks) {
const ancestor = block.content.replaces || block.id;
if (!groups[ancestor]) groups[ancestor] = [];
groups[ancestor].push(block);
}
const liveTipIds = new Set();
for (const groupBlocks of Object.values(groups)) {
let best = groupBlocks[0];
for (const block of groupBlocks) {
if (block.type === 'market') {
if (isClosedSold(block.content.status) && !isClosedSold(best.content.status)) {
best = block;
} else if ((block.content.status === best.content.status) && block.ts > best.ts) {
best = block;
}
} else if (block.type === 'project') {
const br = projectRank(best.content.status);
const cr = projectRank(block.content.status);
if (cr > br || (cr === br && block.ts > best.ts)) best = block;
} else if (block.type === 'job' || block.type === 'forum') {
if (block.ts > best.ts) best = block;
} else {
if (block.ts > best.ts) best = block;
}
}
liveTipIds.add(best.id);
}
const block = idToBlock.get(id);
if (!block) return null;
@ -383,12 +305,8 @@ module.exports = ({ cooler, tribeCrypto, tribesModel }) => {
if (!valid) return null;
}
const c = block.content;
const rootDeleted = c?.type === 'forum' && c.root && tombstoned.has(c.root);
const isTombstoned = tombstoned.has(block.id);
const isReplaced = c.replaces
? (!liveTipIds.has(block.id) || tombstoned.has(block.id))
: referencedAsReplaces.has(block.id) || tombstoned.has(block.id) || rootDeleted;
const isReplaced = isReplacedBlock(versionGroups(versions), tombstoned, block);
return { ...block, isTombstoned, isReplaced };
}

38
src/models/blog_model.js
View file

@ -1,6 +1,8 @@
const pull = require('../server/node_modules/pull-stream');
const { getConfig } = require('../configs/config-manager.js');
const { buildValidatedTombstoneSet } = require('./tombstone_validator');
const { readTyped, memoIndex } = require('./typed_log');
const longText = require('../backend/long_text');
const logLimit = getConfig().ssbLogStream?.limit || 1000;
const OPINION_TYPE = 'blogOpinion';
@ -9,13 +11,9 @@ module.exports = ({ cooler, isPublic = false }) => {
let ssb;
const openSsb = async () => { if (!ssb) ssb = await cooler.open(); return ssb; };
const getAllMessages = async (ssbClient) =>
new Promise((resolve, reject) => {
pull(
ssbClient.createLogStream({ limit: logLimit }),
pull.collect((err, msgs) => (err ? reject(err) : resolve(msgs)))
);
});
const BLOG_TYPES = ['post', OPINION_TYPE, 'tombstone', longText.CHUNK_TYPE];
const getAllMessages = async (ssbClient) => readTyped(ssbClient, BLOG_TYPES, { limit: logLimit });
const isRootPost = (c) =>
!!c && c.type === 'post' && typeof c.text === 'string' &&
@ -23,6 +21,7 @@ module.exports = ({ cooler, isPublic = false }) => {
const buildIndex = (messages) => {
const tomb = buildValidatedTombstoneSet(messages);
const chunks = longText.indexChunks(messages);
const posts = new Map();
const commentsByRoot = new Map();
const opinionsByRoot = new Map();
@ -52,7 +51,7 @@ module.exports = ({ cooler, isPublic = false }) => {
}
}
return { tomb, posts, commentsByRoot, opinionsByRoot };
return { tomb, posts, commentsByRoot, opinionsByRoot, chunks };
};
const buildBlog = (node, idx) => {
@ -63,7 +62,7 @@ module.exports = ({ cooler, isPublic = false }) => {
key: node.key,
author: node.author,
subject: typeof c.contentWarning === 'string' ? c.contentWarning : '',
text: c.text || '',
text: longText.joinText(c.text || '', c.chunks, longText.lookupIn(idx.chunks), node.author),
mentions: Array.isArray(c.mentions) ? c.mentions : [],
allowComments: c.allowComments !== false,
createdAt: new Date(node.ts).toISOString(),
@ -76,7 +75,7 @@ module.exports = ({ cooler, isPublic = false }) => {
const collect = async () => {
const ssbClient = await openSsb();
const messages = await getAllMessages(ssbClient);
const idx = buildIndex(messages);
const idx = memoIndex('blog', messages, buildIndex);
const list = [];
for (const node of idx.posts.values()) list.push(buildBlog(node, idx));
return { list, idx, viewerId: ssbClient.id };
@ -96,6 +95,7 @@ module.exports = ({ cooler, isPublic = false }) => {
if (f === 'MINE') out = out.filter(b => String(b.author) === String(viewerId));
else if (f === 'FAVORITES') out = favorites ? out.filter(b => favorites.has(String(b.id))) : [];
else if (f === 'RECENT') out = out.filter(b => (Date.parse(b.createdAt || '') || 0) >= Date.now() - 86400000);
const q = String(opts.q || '').trim().toLowerCase();
if (q) out = out.filter(b =>
@ -123,14 +123,26 @@ module.exports = ({ cooler, isPublic = false }) => {
const body = String(text || '').trim();
if (!body) throw new Error('Blog text is required');
const ssbClient = await openSsb();
const content = {
const publish = (c) => new Promise((res, rej) => ssbClient.publish(c, (err, msg) => err ? rej(err) : res(msg)));
const content = await longText.chunkContent({
type: 'post',
text: body,
allowComments: allowComments !== false,
...(Array.isArray(mentions) && mentions.length ? { mentions } : {}),
...(String(subject || '').trim() ? { contentWarning: String(subject).trim() } : {})
};
return new Promise((res, rej) => ssbClient.publish(content, (err, msg) => err ? rej(err) : res(msg)));
}, 'text', { publish });
return publish(content);
},
async deleteBlogById(id) {
if (isPublic) throw new Error('Not available in public mode');
const ssbClient = await openSsb();
const userId = ssbClient.id;
const msg = await new Promise((res, rej) => ssbClient.get(id, (err, m) => (err || !m || !m.content) ? rej(new Error('Blog not found')) : res(m)));
if (!isRootPost(msg.content)) throw new Error('Blog not found');
if (msg.author !== userId) throw new Error('Not the author');
const content = { type: 'tombstone', target: id, deletedAt: new Date().toISOString(), author: userId };
return new Promise((res, rej) => ssbClient.publish(content, (err, result) => err ? rej(err) : res(result)));
},
async createOpinion(id, category) {

72
src/models/bookmarking_model.js
View file

@ -3,7 +3,8 @@ const moment = require("../server/node_modules/moment");
const { getConfig } = require("../configs/config-manager.js");
const categories = require("../backend/opinion_categories");
const { buildValidatedTombstoneSet } = require('./tombstone_validator');
const { dedupeBy, norm } = require('../backend/dedupe');
const { readTyped, memoIndex } = require('./typed_log');
const { dedupeBy, norm, versionGroups } = require('../backend/dedupe');
const logLimit = getConfig().ssbLogStream?.limit || 1000;
@ -36,13 +37,9 @@ module.exports = ({ cooler }) => {
return ssb;
};
const getAllMessages = async (ssbClient) =>
new Promise((resolve, reject) => {
pull(
ssbClient.createLogStream({ limit: logLimit }),
pull.collect((err, msgs) => (err ? reject(err) : resolve(msgs)))
);
});
const BOOKMARK_TYPES = ["bookmark", "bookmarkOpinion", "tombstone"];
const getAllMessages = async (ssbClient) => readTyped(ssbClient, BOOKMARK_TYPES, { limit: logLimit });
const getMsg = async (ssbClient, key) =>
new Promise((resolve, reject) => {
@ -52,9 +49,6 @@ module.exports = ({ cooler }) => {
const buildIndex = (messages) => {
const tomb = buildValidatedTombstoneSet(messages);
const nodes = new Map();
const parent = new Map();
const child = new Map();
const strictChild = new Map();
const opinionMsgs = [];
for (const m of messages) {
@ -70,48 +64,16 @@ module.exports = ({ cooler }) => {
const ts = v.timestamp || m.timestamp || 0;
let sizeBytes = 0;
try { sizeBytes = Buffer.byteLength(JSON.stringify(v), "utf8"); } catch (_) { sizeBytes = 0; }
nodes.set(k, { key: k, ts, c, sizeBytes, author: v.author });
if (c.replaces) {
parent.set(k, c.replaces);
child.set(c.replaces, k);
}
nodes.set(k, { key: k, ts, seq: v.sequence || 0, c, sizeBytes, author: v.author });
}
for (const [k, t] of Array.from(parent.entries())) {
const cn = nodes.get(k);
const pn = nodes.get(t);
if (!pn) { parent.delete(k); if (child.get(t) === k) child.delete(t); continue; }
if (!cn || String(cn.author) !== String(pn.author)) { parent.delete(k); if (child.get(t) === k) child.delete(t); nodes.delete(k); }
}
for (const [k, node] of nodes) {
const t = node.c.replaces;
if (t) { const orig = nodes.get(t); if (orig && orig.author === node.author) strictChild.set(t, k); }
}
const rootOf = (id) => {
let cur = id, g = 0;
while (parent.has(cur) && nodes.has(parent.get(cur)) && g++ < 100000) cur = parent.get(cur);
return cur;
};
const contentTipOf = (root) => {
let cur = root, g = 0;
while (strictChild.has(cur) && g++ < 100000) cur = strictChild.get(cur);
const n = nodes.get(cur), rn = nodes.get(root);
return (n && rn && n.author === rn.author) ? cur : root;
};
const roots = new Set();
for (const id of nodes.keys()) roots.add(rootOf(id));
const vg = versionGroups([...nodes.values()].map(n => ({ key: n.key, author: n.author, seq: n.seq, ts: n.ts, replaces: n.c.replaces || null, claimedAuthor: typeof n.c.author === "string" ? n.c.author : null })));
const rootOf = vg.rootOf;
const contentTipOf = vg.tipOf;
const opinionsByRoot = new Map();
for (const op of opinionMsgs) { if (!nodes.has(op.target)) continue; const r = rootOf(op.target); if (!opinionsByRoot.has(r)) opinionsByRoot.set(r, []); opinionsByRoot.get(r).push(op); }
const forward = new Map();
for (const [newId, oldId] of parent.entries()) forward.set(oldId, newId);
const resolveGroup = (root) => {
const contentTip = contentTipOf(root);
const contentNode = nodes.get(contentTip) || nodes.get(root);
@ -128,9 +90,9 @@ module.exports = ({ cooler }) => {
};
const tipByRoot = new Map();
for (const r of roots) tipByRoot.set(r, contentTipOf(r));
for (const r of vg.roots) tipByRoot.set(r, contentTipOf(r));
return { tomb, nodes, parent, child, rootOf, tipByRoot, forward, resolveGroup };
return { tomb, nodes, rootOf, tipByRoot, resolveGroup };
};
const buildBookmark = (node, rootId, viewerId, agg) => {
@ -172,12 +134,8 @@ module.exports = ({ cooler }) => {
const messages = await getAllMessages(ssbClient);
const idx = buildIndex(messages);
let tip = id;
while (idx.forward.has(tip)) tip = idx.forward.get(tip);
if (idx.tomb.has(tip)) throw new Error("Bookmark not found");
let root = tip;
while (idx.parent.has(root)) root = idx.parent.get(root);
const root = idx.rootOf(id);
if (idx.tomb.has(idx.tipByRoot.get(root) || id)) throw new Error("Bookmark not found");
return root;
},
@ -318,10 +276,10 @@ module.exports = ({ cooler }) => {
const viewer = viewerId || ssbClient.id;
const messages = await getAllMessages(ssbClient);
const idx = buildIndex(messages);
const idx = memoIndex('bookmarking', messages, buildIndex);
const rootId = idx.rootOf(id);
const agg = idx.resolveGroup(rootId);
if (!agg.contentNode || idx.tomb.has(agg.contentTip)) throw new Error("Bookmark not found");
if (!idx.tipByRoot.has(rootId) || !agg.contentNode || idx.tomb.has(agg.contentTip)) throw new Error("Bookmark not found");
return buildBookmark(agg.contentNode, rootId, viewer, agg);
},

288
src/models/calendars_model.js
View file

@ -2,6 +2,7 @@ const pull = require("../server/node_modules/pull-stream")
const crypto = require("crypto")
const { getConfig } = require("../configs/config-manager.js")
const { buildValidatedTombstoneSet } = require('./tombstone_validator')
const { readTyped, memoIndex } = require('./typed_log')
const { collabContent, openInviteOf } = require('../backend/collab_content')
const calCollab = collabContent({ membersField: 'participants', undecField: 'encrypted', contentFields: ['title', 'deadline', 'status'], listFields: ['tags', 'invites'] })
const logLimit = getConfig().ssbLogStream?.limit || 1000
@ -14,13 +15,27 @@ const normalizeTags = (raw) => {
return String(raw).split(",").map(t => t.trim()).filter(Boolean)
}
const hasAnyInterval = (w, m, y) => !!(w || m || y)
const { expandRecurrence } = require('./recurrence')
const { expandRecurrence, truthy } = require('./recurrence')
const MAX_OCCURRENCES_PER_DATE = 520
const MAX_DATES_PER_VIEW = 5000
const ts = (v) => {
const t = new Date(v).getTime()
return Number.isFinite(t) ? t : null
}
const boundedOccurrences = (date, until, weekly, monthly, yearly) => {
const start = ts(date)
const end = ts(until)
if (start === null || end === null) return expandRecurrence(date, until, weekly, monthly, yearly).slice(0, MAX_OCCURRENCES_PER_DATE)
const limit = new Date(start)
if (truthy(weekly)) limit.setUTCDate(limit.getUTCDate() + 7 * MAX_OCCURRENCES_PER_DATE)
else if (truthy(monthly)) limit.setUTCMonth(limit.getUTCMonth() + MAX_OCCURRENCES_PER_DATE)
else limit.setUTCFullYear(limit.getUTCFullYear() + MAX_OCCURRENCES_PER_DATE)
const stop = Number.isFinite(limit.getTime()) ? Math.min(end, limit.getTime()) : end
return expandRecurrence(date, new Date(stop).toISOString(), weekly, monthly, yearly).slice(0, MAX_OCCURRENCES_PER_DATE)
}
const assertCalendarDates = ({ deadline, date, until }) => {
const now = Date.now()
const dl = deadline ? ts(deadline) : null
@ -48,6 +63,7 @@ module.exports = ({ cooler, pmModel, tribeCrypto, calendarCrypto, tribesModel })
return (tribeCrypto && tribeCrypto.getKeys(rid)) || []
}
const lookupGen = (rid) => ((ownCrypto && ownCrypto.getGen(rid)) || (tribeCrypto && tribeCrypto.getGen(rid)) || 0)
const keyProofFor = (keyHex, id) => { try { return crypto.createHmac("sha256", Buffer.from(keyHex, "hex")).update(String(id), "utf8").digest("hex") } catch (_) { return null } }
const rotateCalendarKey = async (rootId, remainingMembers) => {
if (!ownCrypto || !tribeCrypto || !rootId) return
@ -55,11 +71,10 @@ module.exports = ({ cooler, pmModel, tribeCrypto, calendarCrypto, tribesModel })
if (!existing) return
const newKey = ownCrypto.generateTribeKey()
const newGen = ownCrypto.addNewKey(rootId, newKey)
if (!Array.isArray(remainingMembers) || !remainingMembers.length) return
const ssbClient = await openSsb()
const ssbKeys = require("../server/node_modules/ssb-keys")
const memberKeys = {}
for (const m of remainingMembers) {
for (const m of new Set([ssbClient.id, ...(Array.isArray(remainingMembers) ? remainingMembers : [])])) {
try { memberKeys[m] = tribeCrypto.boxKeyForMember(newKey, m, ssbKeys) } catch (_) {}
}
if (Object.keys(memberKeys).length) {
@ -76,9 +91,12 @@ module.exports = ({ cooler, pmModel, tribeCrypto, calendarCrypto, tribesModel })
const ssbKeys = require("../server/node_modules/ssb-keys")
const config = require("../server/ssb_config")
const msgs = await readAll(ssbClient)
const authorOf = new Map()
for (const m of msgs) if (m && m.key && m.value) authorOf.set(m.key, m.value.author)
for (const m of msgs) {
const c = m.value && m.value.content
if (!c || c.type !== "tribe-keys") continue
if (!c.tribeId || !authorOf.has(c.tribeId) || (m.value.author !== authorOf.get(c.tribeId) && m.value.author !== ssbClient.id)) continue
const memberKeys = c.memberKeys
if (!memberKeys || typeof memberKeys !== "object") continue
const boxed = memberKeys[ssbClient.id]
@ -92,12 +110,45 @@ module.exports = ({ cooler, pmModel, tribeCrypto, calendarCrypto, tribesModel })
} catch (_) {}
}
const readAll = async (ssbClient) =>
new Promise((resolve, reject) =>
pull(ssbClient.createLogStream({ limit: logLimit }), pull.collect((err, msgs) => err ? reject(err) : resolve(msgs)))
)
const CALENDAR_TYPES = ["calendar", "calendarDate", "calendarNote", "calendarParticipant", "calendarReminderSent", "tribe-keys", "tombstone"]
const readAll = async (ssbClient) => unwrapForIndex(await readTyped(ssbClient, CALENDAR_TYPES, { limit: logLimit, withWindow: true }))
const ensureMemberKeys = async (ssbClient, messages, cals) => {
if (!tribeCrypto || !ownCrypto) return
const leavesByRoot = new Map()
for (const m of messages) {
const c = m.value && m.value.content
if (!c || c.type !== "calendarParticipant" || c.on !== false || !c.target) continue
if (!leavesByRoot.has(c.target)) leavesByRoot.set(c.target, [])
leavesByRoot.get(c.target).push({ member: m.value.author, ts: Number(m.timestamp || m.value.timestamp || 0) })
}
const ssbKeys = require("../server/node_modules/ssb-keys")
for (const cal of (Array.isArray(cals) ? cals : [])) {
if (!cal || cal.tribeId || cal.encrypted || cal.author !== ssbClient.id) continue
const rootId = cal.rootId
if (!rootId || !lookupKey(rootId)) continue
const members = Array.isArray(cal.participants) ? cal.participants : []
const plan = ownCrypto.keyPlan({ rootId, ownerId: cal.author, gen: lookupGen(rootId), members, messages, leaves: leavesByRoot.get(rootId) || [] })
if (plan.rotate) {
await rotateCalendarKey(rootId, members)
continue
}
if (!plan.missing.length) continue
const key = lookupKey(rootId)
const memberKeys = {}
for (const m of plan.missing) {
try { memberKeys[m] = tribeCrypto.boxKeyForMember(key, m, ssbKeys) } catch (_) {}
}
if (!Object.keys(memberKeys).length) continue
await new Promise((resolve) => {
ssbClient.publish({ type: "tribe-keys", tribeId: rootId, generation: lookupGen(rootId) || 1, memberKeys }, () => resolve())
})
}
}
const tribeHelpers = tribeCrypto ? tribeCrypto.createHelpers(tribesModel) : null
const unwrapForIndex = (msgs) => tribeHelpers ? tribeHelpers.unwrapMessagesForKind(msgs, CALENDAR_TYPES) : msgs
const encryptIfTribe = tribeHelpers ? tribeHelpers.encryptIfTribe : async (c) => c
const decryptIfTribe = tribeHelpers ? tribeHelpers.decryptIfTribe : async (c) => c
const assertReadable = tribeHelpers ? tribeHelpers.assertReadable : () => {}
@ -111,7 +162,7 @@ module.exports = ({ cooler, pmModel, tribeCrypto, calendarCrypto, tribesModel })
const decryptScoped = async (content, rootId) => {
if (!content) return content
if (content.tribeId) return await decryptIfTribe(content)
if (content.tribeId || (tribeCrypto && tribeCrypto.isTribeMsg(content))) return await decryptIfTribe(content)
return decryptCalendarRoot(content, rootId)
}
@ -185,9 +236,9 @@ module.exports = ({ cooler, pmModel, tribeCrypto, calendarCrypto, tribesModel })
const c = v.content
if (!c) continue
if (c.type === "tombstone" && c.target) { tombRequests.push({ target: c.target, author: v.author }); continue }
if (c.type === "calendarParticipant" && c.target) { participantMsgs.push({ target: c.target, author: v.author, on: c.on !== false, ts: v.timestamp || m.timestamp || 0 }); continue }
if (c.type === "calendarParticipant" && c.target) { participantMsgs.push({ target: c.target, author: v.author, on: c.on !== false, keyProof: typeof c.keyProof === "string" ? c.keyProof : "", code: typeof c.code === "string" ? c.code : "", ts: v.timestamp || m.timestamp || 0 }); continue }
if (c.type === "calendar") {
nodes.set(k, { key: k, ts: v.timestamp || m.timestamp || 0, c, author: v.author })
nodes.set(k, { key: k, ts: v.timestamp || m.timestamp || 0, seq: v.sequence || 0, c, author: v.author })
authorByKey.set(k, v.author)
if (c.replaces) { parent.set(k, c.replaces); child.set(c.replaces, k) }
}
@ -207,30 +258,31 @@ module.exports = ({ cooler, pmModel, tribeCrypto, calendarCrypto, tribesModel })
const rootOf = (id) => { let cur = id; while (parent.has(cur)) cur = parent.get(cur); return cur }
const tipOf = (id) => { let cur = id; while (child.has(cur)) cur = child.get(cur); return cur }
const contentTipOf = (root) => {
const rn = nodes.get(root)
if (!rn) return root
let cur = root
let best = root
const seen = new Set()
while (child.has(cur) && !seen.has(cur)) {
seen.add(cur)
const next = child.get(cur)
const n = nodes.get(next)
if (!n) break
if (n.author === rn.author && !tomb.has(next)) best = next
cur = next
}
return best
const ownerTip = new Map()
for (const [k, n] of nodes.entries()) {
const r = rootOf(k)
const rn = nodes.get(r)
if (!rn || n.author !== rn.author || tomb.has(k)) continue
const prev = ownerTip.has(r) ? nodes.get(ownerTip.get(r)) : null
if (!prev || n.seq > prev.seq || (n.seq === prev.seq && n.ts >= prev.ts)) ownerTip.set(r, k)
}
const contentTipOf = (root) => ownerTip.get(root) || root
const roots = new Set()
for (const id of nodes.keys()) roots.add(rootOf(id))
const joinAllowed = (root, pm) => {
const rn = nodes.get(root)
if (!rn) return false
if (pm.author === rn.author || rn.c.tribeId) return true
if (!rn.c.encryptedPayload && String(rn.c.status || "OPEN").toUpperCase() === "OPEN") return true
return !!pm.keyProof && lookupKeys(root).some(k => keyProofFor(k, pm.author) === pm.keyProof)
}
const participantsByRoot = new Map()
for (const pm of participantMsgs) {
if (!nodes.has(pm.target)) continue
const r = rootOf(pm.target)
if (pm.on && !joinAllowed(r, pm)) continue
if (!participantsByRoot.has(r)) participantsByRoot.set(r, new Map())
const m2 = participantsByRoot.get(r)
const prev = m2.get(pm.author)
@ -267,17 +319,29 @@ module.exports = ({ cooler, pmModel, tribeCrypto, calendarCrypto, tribesModel })
deadline: undec ? "" : (c.deadline || ""),
tags: Array.isArray(c.tags) ? c.tags : [],
mapUrl: typeof c.mapUrl === "string" ? c.mapUrl : "",
author: c.author || node.author,
author: node.author || c.author,
participants: Array.isArray(participants) ? participants : (Array.isArray(c.participants) ? c.participants : []),
invites: Array.isArray(c.invites) ? c.invites : [],
createdAt: c.createdAt || new Date(node.ts).toISOString(),
updatedAt: c.updatedAt || null,
tribeId: c.tribeId || null,
encrypted: !!undec
encrypted: !!undec,
contentEncrypted: !!(c.encryptedPayload || c._decrypted === true)
}
}
const calendarMembers = async (cal) => {
const set = new Set([cal.author, ...(Array.isArray(cal.participants) ? cal.participants : [])].filter(Boolean))
if (cal.tribeId && tribesModel) {
try {
const t = await tribesModel.getTribeById(cal.tribeId)
for (const m of (t && Array.isArray(t.members) ? t.members : [])) set.add(m)
} catch (_) {}
}
return set
}
const isClosed = (calendar) => {
if (calendar.status === "CLOSED") return true
if (!calendar.deadline) return false
@ -329,11 +393,8 @@ module.exports = ({ cooler, pmModel, tribeCrypto, calendarCrypto, tribesModel })
const ssbClient = await openSsb()
const messages = await readAll(ssbClient)
const idx = buildIndex(messages)
let tip = id
while (idx.child.has(tip)) tip = idx.child.get(tip)
if (idx.tomb.has(tip)) throw new Error("Not found")
let root = tip
while (idx.parent.has(root)) root = idx.parent.get(root)
const root = idx.rootOf(id)
if (idx.tomb.has(idx.contentTipOf(root))) throw new Error("Not found")
return root
},
@ -341,8 +402,7 @@ module.exports = ({ cooler, pmModel, tribeCrypto, calendarCrypto, tribesModel })
const ssbClient = await openSsb()
const messages = await readAll(ssbClient)
const idx = buildIndex(messages)
let tip = id
while (idx.child.has(tip)) tip = idx.child.get(tip)
const tip = idx.contentTipOf(idx.rootOf(id))
if (idx.tomb.has(tip)) throw new Error("Not found")
return tip
},
@ -476,11 +536,9 @@ module.exports = ({ cooler, pmModel, tribeCrypto, calendarCrypto, tribesModel })
ssbClient.get(tipId, (err, it) => err ? reject(err) : resolve(it))
})
if (!item || !item.content) throw new Error("Calendar not found")
const oldDec = item.content.tribeId
? await decryptIfTribe(item.content)
: decryptCalendarRoot(item.content, rootId)
const oldDec = await decryptScoped(item.content, rootId)
assertReadable(oldDec, "Calendar")
if ((oldDec.author || item.content.author) !== userId) throw new Error("Not the author")
if (item.author !== userId) throw new Error("Not the author")
const nextDeadline = data.deadline !== undefined ? data.deadline : (oldDec.deadline || "")
if (nextDeadline !== (oldDec.deadline || "")) assertCalendarDates({ deadline: nextDeadline })
let updated = {
@ -493,12 +551,17 @@ module.exports = ({ cooler, pmModel, tribeCrypto, calendarCrypto, tribesModel })
author: oldDec.author || userId,
participants: oldDec.participants || [userId],
invites: Array.isArray(oldDec.invites) ? oldDec.invites : [],
...(item.content.tribeId ? { tribeId: item.content.tribeId } : {}),
...(oldDec.tribeId ? { tribeId: oldDec.tribeId } : {}),
createdAt: oldDec.createdAt,
updatedAt: new Date().toISOString(),
replaces: tipId
}
if (item.content.tribeId) updated = await encryptIfTribe(updated)
const closing = !oldDec.tribeId && String(oldDec.status || "OPEN").toUpperCase() === "OPEN" && updated.status === "CLOSED"
if (closing) {
updated.invites = updated.invites.filter(inv => !(inv && typeof inv === "object" && inv.public === true))
try { await rotateCalendarKey(rootId, (updated.participants || []).filter(p => p !== userId)) } catch (_) {}
}
if (updated.tribeId) updated = await encryptIfTribe(updated)
else updated = encryptStandalone(updated, rootId)
const result = await new Promise((resolve, reject) => ssbClient.publish(updated, (e, res) => e ? reject(e) : resolve(res)))
const tombstone = { type: "tombstone", target: tipId, deletedAt: new Date().toISOString(), author: userId }
@ -515,8 +578,7 @@ module.exports = ({ cooler, pmModel, tribeCrypto, calendarCrypto, tribesModel })
const rootId = await this.resolveRootId(id)
const dec = await decryptScoped(item.content, rootId)
assertReadable(dec, "Calendar")
const contentAuthor = (dec && dec.author) || (typeof item.content === 'object' && item.content.author)
if (contentAuthor !== userId) throw new Error("Not the author")
if (item.author !== userId) throw new Error("Not the author")
const tombstone = { type: "tombstone", target: tipId, deletedAt: new Date().toISOString(), author: userId }
return new Promise((resolve, reject) => ssbClient.publish(tombstone, e => e ? reject(e) : resolve()))
},
@ -547,9 +609,7 @@ module.exports = ({ cooler, pmModel, tribeCrypto, calendarCrypto, tribesModel })
const participants = Array.isArray(cal.participants) ? cal.participants : []
if (!participants.includes(userId)) return
const content = { type: "calendarParticipant", target: rootId, on: false, createdAt: new Date().toISOString() }
const result = await new Promise((resolve, reject) => ssbClient.publish(content, (e, res) => e ? reject(e) : resolve(res)))
try { await rotateCalendarKey(rootId, participants.filter(p => p !== userId)) } catch (_) {}
return result
return new Promise((resolve, reject) => ssbClient.publish(content, (e, res) => e ? reject(e) : resolve(res)))
},
async findCalendarByLinkText(linkSubstring) {
@ -571,7 +631,7 @@ module.exports = ({ cooler, pmModel, tribeCrypto, calendarCrypto, tribesModel })
async getCalendarById(id) {
const ssbClient = await openSsb()
const messages = await readAll(ssbClient)
const idx = buildIndex(messages)
const idx = memoIndex('calendars', messages, buildIndex)
await decryptIndexNodes(idx)
let root = id
while (idx.parent.has(root)) root = idx.parent.get(root)
@ -589,6 +649,7 @@ module.exports = ({ cooler, pmModel, tribeCrypto, calendarCrypto, tribesModel })
const ssbClient = await openSsb()
const uid = viewerId || ssbClient.id
let list = calCollab.visibleThenCollapsed(await collectCalendars(), uid)
try { await ensureMemberKeys(ssbClient, await readAll(ssbClient), list) } catch (_) {}
if (filter === "mine") list = list.filter(c => c.author === uid)
else if (filter === "recent") {
const now = Date.now()
@ -615,6 +676,10 @@ module.exports = ({ cooler, pmModel, tribeCrypto, calendarCrypto, tribesModel })
})
const ruleDeadline = hasInterval ? (intervalDeadline || cal.deadline || "") : ""
if (!(await calendarMembers(cal)).has(userId)) {
if (cal.tribeId || cal.status !== "OPEN") throw new Error("Only participants can add dates")
await this.joinCalendar(rootId)
}
let dateContent = {
type: "calendarDate",
calendarId: rootId,
@ -640,6 +705,8 @@ module.exports = ({ cooler, pmModel, tribeCrypto, calendarCrypto, tribesModel })
const rootId = await this.resolveRootId(calendarId)
const cal = await this.getCalendarById(rootId)
const calDeadline = cal && cal.deadline ? cal.deadline : ""
const pubKey = cal ? tryDecryptPublicInviteKey(cal.invites) : null
const members = cal ? await calendarMembers(cal) : new Set()
const ssbClient = await openSsb()
const messages = await readAll(ssbClient)
const authorByKey = new Map()
@ -659,6 +726,7 @@ module.exports = ({ cooler, pmModel, tribeCrypto, calendarCrypto, tribesModel })
const c = v.content
if (!c || c.type !== "calendarDate") continue
if (c.calendarId !== rootId) continue
if (!members.has(v.author)) continue
let dec = c
if (c.encryptedPayload && tribeCrypto) {
if (c.tribeId && tribesModel) {
@ -666,25 +734,24 @@ module.exports = ({ cooler, pmModel, tribeCrypto, calendarCrypto, tribesModel })
dec = r && !r._undecryptable ? r : c
if (r && r._undecryptable) continue
} else {
const keys = lookupKeys(c.calendarId)
if (keys && keys.length) {
const r = tribeCrypto.decryptContent(c, keys.map(k => [k]))
dec = r && !r._undecryptable ? r : c
if (r && r._undecryptable) continue
}
const keys = [...(lookupKeys(c.calendarId) || []), ...(pubKey ? [pubKey] : [])]
if (!keys.length) continue
const r = tribeCrypto.decryptContent(c, keys.map(k => [k]))
if (!r || r._undecryptable) continue
dec = r
}
}
const baseEntry = {
key: m.key,
calendarId: dec.calendarId || c.calendarId,
label: dec.label || "",
author: dec.author || v.author,
author: v.author || dec.author,
createdAt: dec.createdAt || new Date(v.timestamp || 0).toISOString()
}
const hasInterval = !!(dec.intervalWeekly || dec.intervalMonthly || dec.intervalYearly)
const ruleDeadline = dec.intervalDeadline || calDeadline
if (hasInterval && ruleDeadline) {
const occurrences = expandRecurrence(dec.date, ruleDeadline, dec.intervalWeekly, dec.intervalMonthly, dec.intervalYearly)
const occurrences = boundedOccurrences(dec.date, ruleDeadline, dec.intervalWeekly, dec.intervalMonthly, dec.intervalYearly)
for (const occ of occurrences) {
dates.push({ ...baseEntry, date: occ.toISOString() })
}
@ -692,8 +759,10 @@ module.exports = ({ cooler, pmModel, tribeCrypto, calendarCrypto, tribesModel })
dates.push({ ...baseEntry, date: dec.date })
}
}
dates.sort((a, b) => new Date(a.date) - new Date(b.date))
return dates
const owner = cal ? cal.author : null
const kept = [...dates.filter(d => d.author === owner), ...dates.filter(d => d.author !== owner)].slice(0, MAX_DATES_PER_VIEW)
kept.sort((a, b) => new Date(a.date) - new Date(b.date))
return kept
},
async deleteDate(dateId, calendarId) {
@ -771,9 +840,7 @@ module.exports = ({ cooler, pmModel, tribeCrypto, calendarCrypto, tribesModel })
const userId = ssbClient.id
const item = await new Promise((resolve, reject) => ssbClient.get(noteId, (e, it) => e ? reject(e) : resolve(it)))
if (!item || !item.content) throw new Error("Note not found")
const noteRoot = item.content.calendarId || null
const dec = await decryptScoped(item.content, noteRoot)
if (((dec && dec.author) || item.content.author) !== userId) throw new Error("Not the author")
if (item.author !== userId) throw new Error("Not the author")
return new Promise((resolve, reject) => {
ssbClient.publish({ type: "tombstone", target: noteId, deletedAt: new Date().toISOString(), author: userId }, (e, msg) => e ? reject(e) : resolve(msg))
})
@ -794,15 +861,23 @@ module.exports = ({ cooler, pmModel, tribeCrypto, calendarCrypto, tribesModel })
}
}
const notes = []
let pubKey
for (const m of messages) {
const v = m.value || {}
const c = v.content
if (!c || c.type !== "calendarNote") continue
if (tombstoned.has(m.key)) continue
if (c.calendarId !== rootId || c.dateId !== dateId) continue
if (c.calendarId !== rootId || (dateId != null && c.dateId !== dateId)) continue
let dec = c
if (c.encryptedPayload) {
const r = await decryptScoped(c, rootId)
let r = await decryptScoped(c, rootId)
if ((!r || r._undecryptable) && !c.tribeId && tribeCrypto) {
if (pubKey === undefined) {
const cal = await this.getCalendarById(rootId).catch(() => null)
pubKey = cal ? tryDecryptPublicInviteKey(cal.invites) : null
}
if (pubKey) r = tribeCrypto.decryptContent(c, [[pubKey]])
}
if (r && !r._undecryptable) dec = r
else continue
}
@ -811,7 +886,7 @@ module.exports = ({ cooler, pmModel, tribeCrypto, calendarCrypto, tribesModel })
calendarId: dec.calendarId || c.calendarId,
dateId: dec.dateId || c.dateId,
text: dec.text || "",
author: dec.author || v.author,
author: v.author || dec.author,
createdAt: dec.createdAt || new Date(v.timestamp || 0).toISOString()
})
}
@ -844,33 +919,36 @@ module.exports = ({ cooler, pmModel, tribeCrypto, calendarCrypto, tribesModel })
}
}
const calendarDeadlines = new Map()
const calendarInfo = new Map()
const dueByCalendar = new Map()
for (const m of messages) {
if (tombstoned.has(m.key)) continue
const dated = messages.filter(m => !tombstoned.has(m.key) && ((m.value || {}).content || {}).type === "calendarDate")
const decoded = await Promise.all(dated.map(async (m) => {
const c = m.value.content
if (!(c.encryptedPayload && tribeCrypto && tribesModel)) return c
const r = await tribeCrypto.decryptFromTribe(c, tribesModel)
return !r || r._undecryptable ? null : r
}))
for (let i = 0; i < dated.length; i++) {
const m = dated[i]
const v = m.value || {}
const c = v.content
if (!c || c.type !== "calendarDate") continue
let dec = c
if (c.encryptedPayload && tribeCrypto && tribesModel) {
const r = await tribeCrypto.decryptFromTribe(c, tribesModel)
if (!r || r._undecryptable) continue
dec = r
}
if (!dec.date) continue
const dec = decoded[i]
if (!dec || !dec.date) continue
const calId = c.calendarId
let calDeadline = calendarDeadlines.get(calId)
if (calDeadline === undefined) {
let info = calendarInfo.get(calId)
if (info === undefined) {
try {
const cc = await this.getCalendarById(calId)
calDeadline = (cc && cc.deadline) || ""
} catch (_) { calDeadline = "" }
calendarDeadlines.set(calId, calDeadline)
info = cc ? { deadline: cc.deadline || "", members: await calendarMembers(cc) } : null
} catch (_) { info = null }
calendarInfo.set(calId, info)
}
if (!info || !info.members.has(v.author)) continue
const calDeadline = info.deadline
const hasInterval = !!(dec.intervalWeekly || dec.intervalMonthly || dec.intervalYearly)
const ruleDeadline = dec.intervalDeadline || calDeadline
const occurrences = (hasInterval && ruleDeadline)
? expandRecurrence(dec.date, ruleDeadline, dec.intervalWeekly, dec.intervalMonthly, dec.intervalYearly)
? boundedOccurrences(dec.date, ruleDeadline, dec.intervalWeekly, dec.intervalMonthly, dec.intervalYearly)
: [new Date(dec.date)]
for (const occ of occurrences) {
if (occ.getTime() > now) continue
@ -934,18 +1012,19 @@ module.exports = ({ cooler, pmModel, tribeCrypto, calendarCrypto, tribesModel })
if (cal.author !== userId) throw new Error("Only the author can generate invites")
const code = crypto.randomBytes(INVITE_CODE_BYTES).toString("hex")
let invite = code
const pubFlag = opts.public ? { public: true } : {}
if (tribeCrypto && !cal.tribeId) {
const calKey = !cal.tribeId ? lookupKey(cal.rootId) : null
if (tribeCrypto && calKey) {
const inviteSalt = tribeCrypto.generateInviteSalt()
const ekChain = tribeCrypto.encryptChainForInvite([cal.rootId], code, inviteSalt)
if (ekChain) invite = { code, ekChain, salt: inviteSalt, gen: lookupGen(cal.rootId), ...pubFlag }
const ek = tribeCrypto.encryptForInvite(calKey, code, inviteSalt)
invite = opts.public
? { code, ek, salt: inviteSalt, gen: lookupGen(cal.rootId), public: true }
: { ch: tribeCrypto.hashInviteCode(code, inviteSalt), ek, salt: inviteSalt, gen: lookupGen(cal.rootId) }
}
if (opts.public && typeof invite !== "object") invite = { code, public: true }
const tipId = await this.resolveCurrentId(calendarId)
const item = await new Promise((resolve, reject) => ssbClient.get(tipId, (e, it) => e ? reject(e) : resolve(it)))
const dec = item.content.tribeId
? await decryptIfTribe(item.content)
: decryptCalendarRoot(item.content, cal.rootId)
const dec = await decryptScoped(item.content, cal.rootId)
assertReadable(dec, "Calendar")
const invites = [...(Array.isArray(dec.invites) ? dec.invites : []), invite]
let updated = {
type: "calendar",
@ -957,12 +1036,12 @@ module.exports = ({ cooler, pmModel, tribeCrypto, calendarCrypto, tribesModel })
author: dec.author,
participants: Array.isArray(dec.participants) ? dec.participants : [userId],
invites,
...(item.content.tribeId ? { tribeId: item.content.tribeId } : {}),
...(dec.tribeId ? { tribeId: dec.tribeId } : {}),
createdAt: dec.createdAt,
updatedAt: new Date().toISOString(),
replaces: tipId
}
if (item.content.tribeId) updated = await encryptIfTribe(updated)
if (updated.tribeId) updated = await encryptIfTribe(updated)
else updated = encryptStandalone(updated, cal.rootId)
await new Promise((resolve, reject) => ssbClient.publish(updated, (e, res) => e ? reject(e) : resolve(res)))
const tombstone = { type: "tombstone", target: tipId, deletedAt: new Date().toISOString(), author: userId }
@ -990,9 +1069,8 @@ module.exports = ({ cooler, pmModel, tribeCrypto, calendarCrypto, tribesModel })
if (cal.author !== userId) throw new Error("Only the author can remove invites")
const tipId = await this.resolveCurrentId(calendarId)
const item = await new Promise((resolve, reject) => ssbClient.get(tipId, (e, it) => e ? reject(e) : resolve(it)))
const dec = item.content.tribeId
? await decryptIfTribe(item.content)
: decryptCalendarRoot(item.content, cal.rootId)
const dec = await decryptScoped(item.content, cal.rootId)
assertReadable(dec, "Calendar")
const invites = (Array.isArray(dec.invites) ? dec.invites : []).filter(inv => !(typeof inv === "object" && inv.public === true))
let updated = {
type: "calendar",
@ -1004,16 +1082,17 @@ module.exports = ({ cooler, pmModel, tribeCrypto, calendarCrypto, tribesModel })
author: dec.author,
participants: Array.isArray(dec.participants) ? dec.participants : [userId],
invites,
...(item.content.tribeId ? { tribeId: item.content.tribeId } : {}),
...(dec.tribeId ? { tribeId: dec.tribeId } : {}),
createdAt: dec.createdAt,
updatedAt: new Date().toISOString(),
replaces: tipId
}
if (item.content.tribeId) updated = await encryptIfTribe(updated)
if (updated.tribeId) updated = await encryptIfTribe(updated)
else updated = encryptStandalone(updated, cal.rootId)
await new Promise((resolve, reject) => ssbClient.publish(updated, (e, res) => e ? reject(e) : resolve(res)))
const tombstone = { type: "tombstone", target: tipId, deletedAt: new Date().toISOString(), author: userId }
await new Promise((resolve, reject) => ssbClient.publish(tombstone, e => e ? reject(e) : resolve()))
if (!dec.tribeId) await rotateCalendarKey(cal.rootId, Array.isArray(cal.participants) ? cal.participants : [])
},
async joinByInvite(code) {
@ -1026,12 +1105,22 @@ module.exports = ({ cooler, pmModel, tribeCrypto, calendarCrypto, tribesModel })
const invs = Array.isArray(cal.invites) ? cal.invites : []
for (const inv of invs) {
if (typeof inv === "string" && inv === code) { matched = cal; matchedInvite = inv; break }
if (typeof inv === "object" && inv.code === code) { matched = cal; matchedInvite = inv; break }
if (typeof inv === "object" && inv && (inv.code === code || (inv.ch && tribeCrypto && inv.ch === tribeCrypto.hashInviteCode(code, inv.salt)))) { matched = cal; matchedInvite = inv; break }
}
if (matched) break
}
if (!matched) throw new Error("Invalid or expired invite code")
if (matched.participants.includes(userId)) throw new Error("Already a participant")
if (typeof matchedInvite === "object" && matchedInvite.ch && tribeCrypto && matchedInvite.ek) {
let probeKey = null
try { probeKey = tribeCrypto.decryptFromInvite(matchedInvite.ek, code, matchedInvite.salt) } catch (_) {}
const messages = await readAll(await openSsb())
const spent = messages.some(m => {
const c = m.value && m.value.content
return c && c.type === "calendarParticipant" && c.on !== false && c.target === matched.rootId && c.code === matchedInvite.ch && m.value.author !== userId && (!probeKey || keyProofFor(probeKey, m.value.author) === c.keyProof)
})
if (spent) throw new Error("Invite already used")
}
let calKey = null
if (tribeCrypto && typeof matchedInvite === "object") {
if (matchedInvite.ekChain) {
@ -1053,9 +1142,8 @@ module.exports = ({ cooler, pmModel, tribeCrypto, calendarCrypto, tribesModel })
}
const tipId = await this.resolveCurrentId(matched.rootId)
const item = await new Promise((resolve, reject) => ssbClient.get(tipId, (e, it) => e ? reject(e) : resolve(it)))
const dec = item.content.tribeId
? await decryptIfTribe(item.content)
: decryptCalendarRoot(item.content, matched.rootId)
const dec = await decryptScoped(item.content, matched.rootId)
assertReadable(dec, "Calendar")
const isPublicInvite = typeof matchedInvite === "object" && matchedInvite.public === true
const invites = isPublicInvite
? (Array.isArray(dec.invites) ? dec.invites : [])
@ -1073,12 +1161,12 @@ module.exports = ({ cooler, pmModel, tribeCrypto, calendarCrypto, tribesModel })
author: dec.author,
participants: [...(Array.isArray(dec.participants) ? dec.participants : []), userId],
invites,
...(item.content.tribeId ? { tribeId: item.content.tribeId } : {}),
...(dec.tribeId ? { tribeId: dec.tribeId } : {}),
createdAt: dec.createdAt,
updatedAt: new Date().toISOString(),
replaces: tipId
}
if (item.content.tribeId) updated = await encryptIfTribe(updated)
if (updated.tribeId) updated = await encryptIfTribe(updated)
else updated = encryptStandalone(updated, matched.rootId)
await new Promise((resolve, reject) => ssbClient.publish(updated, (e, res) => e ? reject(e) : resolve(res)))
const tombstone = { type: "tombstone", target: tipId, deletedAt: new Date().toISOString(), author: userId }
@ -1098,7 +1186,7 @@ module.exports = ({ cooler, pmModel, tribeCrypto, calendarCrypto, tribesModel })
}
} catch (_) {}
}
await new Promise((resolve, reject) => ssbClient.publish({ type: "calendarParticipant", target: matched.rootId, on: true, createdAt: new Date().toISOString() }, (e, res) => e ? reject(e) : resolve(res)))
await new Promise((resolve, reject) => ssbClient.publish({ type: "calendarParticipant", target: matched.rootId, on: true, ...(calKey ? { keyProof: keyProofFor(calKey, userId) } : {}), ...(matchedInvite && typeof matchedInvite === "object" && matchedInvite.ch ? { code: matchedInvite.ch } : {}), createdAt: new Date().toISOString() }, (e, res) => e ? reject(e) : resolve(res)))
return matched.rootId
}
}

312
src/models/campaigns_model.js Normal file
View file

@ -0,0 +1,312 @@
const { readTyped, memoIndex } = require('./typed_log');
const categories = require('../backend/opinion_categories');
const { buildValidatedTombstoneSet } = require('./tombstone_validator');
const { getConfig } = require('../configs/config-manager.js');
const logLimit = getConfig().ssbLogStream?.limit || 1000;
const CAMPAIGN_TYPE = 'campaign';
const SIGNATURE_TYPE = 'campaignSignature';
const UPDATE_TYPE = 'campaignUpdate';
const OPINION_TYPE = 'campaignOpinion';
const PROPOSAL_TYPE = 'parliamentProposal';
const TYPES = [CAMPAIGN_TYPE, SIGNATURE_TYPE, UPDATE_TYPE, OPINION_TYPE, PROPOSAL_TYPE, 'tombstone'];
const CATEGORIES = ['ENVIRONMENT', 'RIGHTS', 'HEALTH', 'EDUCATION', 'INFRASTRUCTURE', 'CULTURE', 'ECONOMY', 'OTHER'];
const STATUSES = ['OPEN', 'ACHIEVED', 'CLOSED'];
const RECENT_MS = 24 * 60 * 60 * 1000;
const safeText = (v) => String(v == null ? '' : v).trim();
const normU = (v) => String(v || '').trim().toUpperCase();
const normalizeCategory = (v) => (CATEGORIES.includes(normU(v)) ? normU(v) : 'OTHER');
const normalizeTags = (raw) => {
if (!raw) return [];
const arr = Array.isArray(raw) ? raw : String(raw).split(',');
return Array.from(new Set(arr.map(t => String(t || '').trim()).filter(Boolean)));
};
const toIso = (v) => { const t = Date.parse(v || ''); return Number.isFinite(t) ? new Date(t).toISOString() : ''; };
const MEDIA_RE = /!?\[(image|video):[^\]]*\]\((&[^)]+)\)/;
const mediaOf = (text) => { const m = safeText(text).match(MEDIA_RE); return m ? { kind: m[1], blobId: m[2] } : null; };
const goalOf = (v) => { const n = Math.floor(Number(v)); return Number.isFinite(n) && n > 0 ? n : 0; };
module.exports = ({ cooler }) => {
let ssb;
const openSsb = async () => { if (!ssb) ssb = await cooler.open(); return ssb; };
const publish = (ssbClient, content) => new Promise((resolve, reject) => ssbClient.publish(content, (err, res) => err ? reject(err) : resolve(res)));
const buildIndex = (messages) => {
const tomb = buildValidatedTombstoneSet(messages);
const nodes = new Map();
const parentOf = new Map();
const signatures = [];
const updates = [];
const opinions = [];
const proposalByCampaign = new Map();
for (const m of messages) {
const v = m.value || {};
const c = v.content;
if (!c || typeof c !== 'object') continue;
if (c.type === CAMPAIGN_TYPE) {
nodes.set(m.key, { key: m.key, author: v.author, ts: v.timestamp || m.timestamp || 0, c });
if (typeof c.replaces === 'string') parentOf.set(m.key, c.replaces);
} else if (c.type === SIGNATURE_TYPE && typeof c.target === 'string') {
signatures.push({ key: m.key, target: c.target, author: v.author, ts: v.timestamp || 0, text: safeText(c.text), createdAt: c.createdAt || new Date(v.timestamp || 0).toISOString() });
} else if (c.type === PROPOSAL_TYPE) {
const legacy = typeof c.campaignId === 'string' && c.campaignId ? null : String(c.description || '').match(/signatures: \/campaigns\/(%[^\s]+\.sha256)\s*$/);
const ref = typeof c.campaignId === 'string' && c.campaignId ? c.campaignId : (legacy ? legacy[1] : '');
if (ref && !proposalByCampaign.has(ref)) proposalByCampaign.set(ref, m.key);
} else if (c.type === OPINION_TYPE && typeof c.target === 'string') {
opinions.push({ target: c.target, author: v.author, category: String(c.category || ''), ts: v.timestamp || 0 });
} else if (c.type === UPDATE_TYPE && typeof c.target === 'string') {
updates.push({ key: m.key, target: c.target, author: v.author, ts: v.timestamp || 0, text: safeText(c.text), createdAt: c.createdAt || new Date(v.timestamp || 0).toISOString(), replaces: typeof c.replaces === 'string' ? c.replaces : null });
}
}
const updateParent = new Map(updates.filter(u => u.replaces).map(u => [u.key, u.replaces]));
const updateRootOf = (key) => {
const seen = new Set();
let cur = key;
while (updateParent.has(cur) && !seen.has(cur)) { seen.add(cur); cur = updateParent.get(cur); }
return cur;
};
const updateVersions = new Map();
for (const u of updates) {
const uroot = updateRootOf(u.key);
if (tomb.has(uroot)) continue;
if (!updateVersions.has(uroot)) updateVersions.set(uroot, []);
updateVersions.get(uroot).push(u);
}
const updateItems = [];
for (const [uroot, versions] of updateVersions) {
versions.sort((a, b) => a.ts - b.ts);
const first = versions[0];
const own = versions.filter(v => v.author === first.author);
const tip = own[own.length - 1];
updateItems.push({ id: uroot, key: uroot, tipId: tip.key, target: first.target, author: first.author, ts: first.ts, text: tip.text, createdAt: first.createdAt, updatedAt: tip.createdAt, edited: tip.key !== uroot });
}
const rootOf = (key) => {
const seen = new Set();
let cur = key;
while (parentOf.has(cur) && !seen.has(cur)) { seen.add(cur); cur = parentOf.get(cur); }
return cur;
};
const versionsByRoot = new Map();
for (const node of nodes.values()) {
const root = rootOf(node.key);
if (tomb.has(root)) continue;
if (!versionsByRoot.has(root)) versionsByRoot.set(root, []);
versionsByRoot.get(root).push(node);
}
const sigByRoot = new Map();
for (const s of signatures) {
if (!nodes.has(s.target)) continue;
const r = rootOf(s.target);
if (!sigByRoot.has(r)) sigByRoot.set(r, new Map());
const prev = sigByRoot.get(r).get(s.author);
if (!prev || s.ts < prev.ts) sigByRoot.get(r).set(s.author, s);
}
const opinionsByRoot = new Map();
for (const op of opinions) {
if (!nodes.has(op.target)) continue;
const r = rootOf(op.target);
if (!opinionsByRoot.has(r)) opinionsByRoot.set(r, []);
opinionsByRoot.get(r).push(op);
}
const campaigns = new Map();
const now = Date.now();
for (const [root, versions] of versionsByRoot) {
versions.sort((a, b) => a.ts - b.ts);
const first = versions[0];
const own = versions.filter(v => v.author === first.author);
const tip = own[own.length - 1];
const c = tip.c;
const sigs = Array.from((sigByRoot.get(root) || new Map()).values()).sort((a, b) => a.ts - b.ts);
const goal = goalOf(c.goal);
const deadline = toIso(c.deadline);
const deadlineTs = Date.parse(deadline) || 0;
const expired = !!deadlineTs && deadlineTs < now;
const achieved = goal > 0 && sigs.length >= goal;
const manual = normU(c.status) === 'CLOSED';
const status = manual ? 'CLOSED' : (achieved ? 'ACHIEVED' : (expired ? 'CLOSED' : 'OPEN'));
const thread = updateItems.filter(u => nodes.has(u.target) && rootOf(u.target) === root && u.author === first.author).sort((a, b) => a.ts - b.ts);
const ops = {};
const voters = [];
for (const op of (opinionsByRoot.get(root) || []).sort((a, b) => a.ts - b.ts)) {
if (voters.includes(op.author) || !categories.includes(op.category)) continue;
voters.push(op.author);
ops[op.category] = (ops[op.category] || 0) + 1;
}
campaigns.set(root, {
id: root, rootId: root, tipId: tip.key,
title: safeText(c.title), text: safeText(c.text), media: mediaOf(c.text), category: normalizeCategory(c.category),
goal, deadline, deadlineTs, expired, tags: normalizeTags(c.tags), mapUrl: safeText(c.mapUrl),
status, achieved, closed: status === 'CLOSED', proposalId: safeText(c.proposalId) || proposalByCampaign.get(root) || '',
author: first.author, createdAt: first.c.createdAt || new Date(first.ts).toISOString(),
updatedAt: c.updatedAt || new Date(tip.ts).toISOString(), ts: tip.ts,
signatures: sigs, signatureCount: sigs.length, signers: sigs.map(s => s.author),
progress: goal > 0 ? Math.min(100, Math.round((sigs.length / goal) * 100)) : 0,
updates: thread,
opinions: ops, opinions_inhabitants: voters, opinionCount: voters.length,
lastActivityTs: Math.max(tip.ts, ...sigs.map(s => s.ts), ...thread.map(u => u.ts))
});
}
return { campaigns, rootOf, nodes, updatesById: new Map(updateItems.map(u => [u.id, u])), updateRootOf };
};
const load = async () => {
const ssbClient = await openSsb();
const idx = memoIndex('campaigns', await readTyped(ssbClient, TYPES, { limit: logLimit }), buildIndex);
return { ssbClient, idx };
};
const find = (idx, id) => {
const key = String(id || '');
if (idx.campaigns.has(key)) return idx.campaigns.get(key);
if (idx.nodes.has(key)) return idx.campaigns.get(idx.rootOf(key)) || null;
return null;
};
const decorate = (cp, me) => ({
...cp,
isOwner: cp.author === me,
signed: cp.signers.includes(me),
canSign: !cp.closed && !cp.signers.includes(me),
canElevate: cp.author === me && cp.achieved && !cp.proposalId
});
const versionOf = (cp, me, data = {}) => ({
type: CAMPAIGN_TYPE,
title: safeText(data.title) || cp.title,
text: data.text !== undefined ? safeText(data.text) : cp.text,
category: data.category !== undefined ? normalizeCategory(data.category) : cp.category,
goal: data.goal !== undefined && goalOf(data.goal) ? goalOf(data.goal) : cp.goal,
deadline: data.deadline !== undefined ? toIso(data.deadline) : cp.deadline,
tags: data.tags !== undefined ? normalizeTags(data.tags) : cp.tags,
mapUrl: data.mapUrl !== undefined ? safeText(data.mapUrl) : cp.mapUrl,
status: data.status !== undefined ? (normU(data.status) === 'CLOSED' ? 'CLOSED' : 'OPEN') : (cp.status === 'CLOSED' && !cp.expired ? 'CLOSED' : 'OPEN'),
proposalId: data.proposalId !== undefined ? safeText(data.proposalId) : cp.proposalId,
author: me,
createdAt: cp.createdAt,
updatedAt: new Date().toISOString(),
replaces: cp.tipId
});
return {
CATEGORIES,
STATUSES,
async createCampaign({ title, text, category, goal, deadline, tags, mapUrl }) {
const ssbClient = await openSsb();
const t = safeText(title);
if (!t) throw new Error('Title is required');
const g = goalOf(goal);
if (!g) throw new Error('A signature goal is required');
const now = new Date().toISOString();
return publish(ssbClient, {
type: CAMPAIGN_TYPE, title: t, text: safeText(text), category: normalizeCategory(category), goal: g,
deadline: toIso(deadline), tags: normalizeTags(tags), mapUrl: safeText(mapUrl), status: 'OPEN',
author: ssbClient.id, createdAt: now, updatedAt: now
});
},
async updateCampaign(id, data = {}) {
const { ssbClient, idx } = await load();
const cp = find(idx, id);
if (!cp) throw new Error('Campaign not found');
if (cp.author !== ssbClient.id) throw new Error('Only the promoter can update this campaign');
const res = await publish(ssbClient, versionOf(cp, ssbClient.id, data));
return { key: res.key, rootId: cp.id };
},
async closeCampaign(id) { return this.updateCampaign(id, { status: 'CLOSED' }); },
async reopenCampaign(id) { return this.updateCampaign(id, { status: 'OPEN' }); },
async deleteCampaign(id) {
const { ssbClient, idx } = await load();
const cp = find(idx, id);
if (!cp) throw new Error('Campaign not found');
if (cp.author !== ssbClient.id) throw new Error('Only the promoter can delete this campaign');
await publish(ssbClient, { type: 'tombstone', target: cp.id, deletedAt: new Date().toISOString(), author: ssbClient.id });
return { key: cp.id };
},
async sign(id, text = '') {
const { ssbClient, idx } = await load();
const cp = find(idx, id);
if (!cp) throw new Error('Campaign not found');
if (cp.closed) throw new Error('This campaign is closed');
if (cp.signers.includes(ssbClient.id)) throw new Error('You already signed this campaign');
return publish(ssbClient, { type: SIGNATURE_TYPE, target: cp.id, text: safeText(text).slice(0, 500), createdAt: new Date().toISOString() });
},
async addUpdate(id, text) {
const { ssbClient, idx } = await load();
const cp = find(idx, id);
if (!cp) throw new Error('Campaign not found');
if (cp.author !== ssbClient.id) throw new Error('Only the promoter can post updates');
const body = safeText(text);
if (!body) throw new Error('Text is required');
return publish(ssbClient, { type: UPDATE_TYPE, target: cp.id, text: body, createdAt: new Date().toISOString() });
},
async editUpdate(updateId, text) {
const { ssbClient, idx } = await load();
const update = idx.updatesById.get(idx.updateRootOf(String(updateId || '')));
if (!update) throw new Error('Update not found');
if (String(update.author) !== String(ssbClient.id)) throw new Error('Only the promoter can edit an update');
const body = safeText(text);
if (!body) throw new Error('Text is required');
const res = await publish(ssbClient, { type: UPDATE_TYPE, target: update.target, text: body, createdAt: new Date().toISOString(), replaces: update.tipId });
return { key: res.key, rootId: update.id, campaignId: update.target };
},
async deleteUpdate(updateId) {
const { ssbClient, idx } = await load();
const update = idx.updatesById.get(idx.updateRootOf(String(updateId || '')));
if (!update) throw new Error('Update not found');
if (String(update.author) !== String(ssbClient.id)) throw new Error('Only the promoter can delete an update');
await publish(ssbClient, { type: 'tombstone', target: update.id, deletedAt: new Date().toISOString(), author: ssbClient.id });
return { key: update.id, campaignId: update.target };
},
async createOpinion(id, category) {
if (!categories.includes(category)) throw new Error('Invalid voting category');
const { ssbClient, idx } = await load();
const cp = find(idx, id);
if (!cp) throw new Error('Campaign not found');
if (cp.opinions_inhabitants.includes(ssbClient.id)) throw new Error('Already voted');
return publish(ssbClient, { type: OPINION_TYPE, target: cp.id, category, createdAt: new Date().toISOString() });
},
async recordProposal(id, proposalId) {
return this.updateCampaign(id, { proposalId: safeText(proposalId) });
},
async getCampaignById(id, viewerId = null) {
const { ssbClient, idx } = await load();
const cp = find(idx, id);
return cp ? decorate(cp, viewerId || ssbClient.id) : null;
},
async listAll({ filter = 'all', q = '', viewerId = null } = {}) {
const { ssbClient, idx } = await load();
const me = viewerId || ssbClient.id;
const f = String(filter || 'all').toLowerCase();
const needle = String(q || '').trim().toLowerCase();
let out = Array.from(idx.campaigns.values()).map(cp => decorate(cp, me));
if (f === 'mine') out = out.filter(cp => cp.isOwner);
else if (f === 'signed') out = out.filter(cp => cp.signed);
else if (f === 'recent') out = out.filter(cp => cp.lastActivityTs >= Date.now() - RECENT_MS);
else if (f === 'top') out = out.filter(cp => cp.signatureCount > 0);
else if (STATUSES.includes(f.toUpperCase())) out = out.filter(cp => cp.status === f.toUpperCase());
else if (CATEGORIES.includes(f.toUpperCase())) out = out.filter(cp => cp.category === f.toUpperCase());
if (needle) out = out.filter(cp => [cp.title, cp.text, cp.category, ...cp.tags].some(v => String(v || '').toLowerCase().includes(needle)));
if (f === 'top') out.sort((a, b) => b.signatureCount - a.signatureCount || b.lastActivityTs - a.lastActivityTs);
else out.sort((a, b) => b.lastActivityTs - a.lastActivityTs);
return out;
},
async resolveRootId(id) {
const { idx } = await load();
const cp = find(idx, id);
return cp ? cp.id : String(id || '');
}
};
};

354
src/models/chats_model.js
View file

@ -2,6 +2,7 @@ const pull = require("../server/node_modules/pull-stream")
const crypto = require("crypto")
const { getConfig } = require("../configs/config-manager.js")
const { buildValidatedTombstoneSet } = require('./tombstone_validator')
const { readTyped, memoIndex } = require("./typed_log")
const { collabContent, openInviteOf } = require('../backend/collab_content')
const chatCollab = collabContent({ membersField: 'members', undecField: 'undecryptable', contentFields: ['title', 'description', 'image', 'category', 'status'], listFields: ['tags', 'invites'] })
const logLimit = getConfig().ssbLogStream?.limit || 1000
@ -16,6 +17,7 @@ const normalizeTags = (raw) => {
const INVITE_CODE_BYTES = 16
const VALID_STATUS = ["OPEN", "INVITE-ONLY", "CLOSED"]
const REACTION_EMOJIS = ["up", "heart", "laugh", "down"]
const DEFAULT_MESSAGES_PER_HOUR = 60
@ -35,6 +37,17 @@ module.exports = ({ cooler, tribeCrypto, chatCrypto, tribesModel }) => {
const ownCrypto = chatCrypto || tribeCrypto
const lookupKey = (rid) => (ownCrypto && ownCrypto.getKey(rid)) || (tribeCrypto && tribeCrypto.getKey(rid)) || null
const hashCode = (code, salt) => tribeCrypto ? tribeCrypto.hashInviteCode(code, salt) : null
const inviteMatches = (inv, token) => {
if (!token) return false
if (typeof inv === "string") return inv === token
if (!inv || typeof inv !== "object") return false
if (typeof inv.code === "string" && inv.code === token) return true
if (typeof inv.ch === "string" && (inv.ch === token || hashCode(token, inv.salt) === inv.ch)) return true
return false
}
const inviteToken = (inv) => (inv && typeof inv === "object" && typeof inv.ch === "string") ? inv.ch : (typeof inv === "string" ? inv : (inv && inv.code) || "")
const keyProofFor = (keyHex, id) => { try { return crypto.createHmac("sha256", Buffer.from(keyHex, "hex")).update(String(id), "utf8").digest("hex") } catch (_) { return null } }
const lookupKeys = (rid) => {
const a = (ownCrypto && ownCrypto.getKeys(rid)) || []
if (a.length) return a
@ -48,11 +61,10 @@ module.exports = ({ cooler, tribeCrypto, chatCrypto, tribesModel }) => {
if (!existing) return
const newKey = ownCrypto.generateTribeKey()
const newGen = ownCrypto.addNewKey(rootId, newKey)
if (!Array.isArray(remainingMembers) || !remainingMembers.length) return
const ssbClient = await openSsb()
const ssbKeys = require("../server/node_modules/ssb-keys")
const memberKeys = {}
for (const m of remainingMembers) {
for (const m of new Set([ssbClient.id, ...(Array.isArray(remainingMembers) ? remainingMembers : [])])) {
try { memberKeys[m] = tribeCrypto.boxKeyForMember(newKey, m, ssbKeys) } catch (_) {}
}
if (Object.keys(memberKeys).length) {
@ -69,9 +81,15 @@ module.exports = ({ cooler, tribeCrypto, chatCrypto, tribesModel }) => {
const ssbKeys = require("../server/node_modules/ssb-keys")
const config = require("../server/ssb_config")
const msgs = await readAll(ssbClient)
const idx = buildIndex(msgs)
const trusted = (root, author) => {
const n = idx.nodes.get(root)
return !!author && !!n && (author === n.author || author === ssbClient.id)
}
for (const m of msgs) {
const c = m.value && m.value.content
if (!c || c.type !== "tribe-keys") continue
if (!trusted(c.tribeId, m.value.author)) continue
const memberKeys = c.memberKeys
if (!memberKeys || typeof memberKeys !== "object") continue
const boxed = memberKeys[ssbClient.id]
@ -98,10 +116,9 @@ module.exports = ({ cooler, tribeCrypto, chatCrypto, tribesModel }) => {
return ks.length ? ks[0] : null
}
const readAll = async (ssbClient) =>
new Promise((resolve, reject) =>
pull(ssbClient.createLogStream({ limit: logLimit }), pull.collect((err, msgs) => err ? reject(err) : resolve(msgs)))
)
const CHAT_TYPES = ["chat", "chatMessage", "chatMember", "chatReaction", "chatPin", "tribe-keys", "tombstone"]
const readAll = async (ssbClient) => readTyped(ssbClient, CHAT_TYPES, { limit: logLimit })
const buildIndex = (messages) => {
const tomb = new Set()
@ -112,6 +129,8 @@ module.exports = ({ cooler, tribeCrypto, chatCrypto, tribesModel }) => {
const authorByKey = new Map()
const tombRequests = []
const memberMsgs = []
const reactMsgs = []
const pinMsgs = []
for (const m of messages) {
const k = m.key
@ -120,11 +139,19 @@ module.exports = ({ cooler, tribeCrypto, chatCrypto, tribesModel }) => {
if (!c) continue
if (c.type === "tombstone" && c.target) { tombRequests.push({ target: c.target, author: v.author }); continue }
if (c.type === "chatMember" && c.target && c.member) {
memberMsgs.push({ target: c.target, member: c.member, on: c.on !== false, code: typeof c.code === "string" ? c.code : "", author: v.author, ts: v.timestamp || m.timestamp || 0 })
memberMsgs.push({ target: c.target, member: c.member, on: c.on !== false, code: typeof c.code === "string" ? c.code : "", keyProof: typeof c.keyProof === "string" ? c.keyProof : "", author: v.author, ts: v.timestamp || m.timestamp || 0 })
continue
}
if (c.type === "chatReaction" && c.target && REACTION_EMOJIS.includes(c.emoji)) {
reactMsgs.push({ target: c.target, emoji: c.emoji, on: c.on !== false, author: v.author, ts: v.timestamp || m.timestamp || 0 })
continue
}
if (c.type === "chatPin" && c.target) {
pinMsgs.push({ target: c.target, on: c.on !== false, author: v.author, ts: v.timestamp || m.timestamp || 0 })
continue
}
if (c.type === "chat") {
nodes.set(k, { key: k, ts: v.timestamp || m.timestamp || 0, c, author: v.author })
nodes.set(k, { key: k, ts: v.timestamp || m.timestamp || 0, seq: v.sequence || 0, c, author: v.author })
authorByKey.set(k, v.author)
if (c.replaces) { parent.set(k, c.replaces); child.set(c.replaces, k) }
} else if (c.type === "chatMessage") {
@ -133,24 +160,28 @@ module.exports = ({ cooler, tribeCrypto, chatCrypto, tribesModel }) => {
}
}
const strictParent = new Map()
const strictChild = new Map()
for (const [c1, p1] of parent.entries()) {
const childNode = nodes.get(c1)
const parentNode = nodes.get(p1)
if (childNode && parentNode && childNode.author === parentNode.author) {
strictParent.set(c1, p1)
strictChild.set(p1, c1)
}
}
for (const t of tombRequests) {
const targetAuthor = authorByKey.get(t.target)
if (targetAuthor && t.author === targetAuthor) tomb.add(t.target)
}
const rootOf = (id) => { let cur = id; while (strictParent.has(cur)) cur = strictParent.get(cur); return cur }
const tipOf = (id) => { let cur = id; while (strictChild.has(cur)) cur = strictChild.get(cur); return cur }
const rootMemo = new Map()
const rootOf = (id) => {
if (rootMemo.has(id)) return rootMemo.get(id)
let cur = id, g = 0
while (parent.has(cur) && nodes.has(parent.get(cur)) && g++ < 100000) cur = parent.get(cur)
rootMemo.set(id, cur)
return cur
}
const ownerTip = new Map()
for (const [k, n] of nodes) {
const r = rootOf(k)
const rn = nodes.get(r)
if (!rn || n.author !== rn.author) continue
const prev = ownerTip.has(r) ? nodes.get(ownerTip.get(r)) : null
if (!prev || n.seq > prev.seq || (n.seq === prev.seq && n.ts >= prev.ts)) ownerTip.set(r, k)
}
const tipOf = (id) => { const r = rootOf(id); return ownerTip.get(r) || r }
const roots = new Set()
for (const id of nodes.keys()) roots.add(rootOf(id))
@ -158,25 +189,72 @@ module.exports = ({ cooler, tribeCrypto, chatCrypto, tribesModel }) => {
for (const r of roots) tipByRoot.set(r, tipOf(r))
const memberByRoot = new Map()
const invitesOf = (root) => {
const out = []
for (const id of [root, tipOf(root)]) { const n = nodes.get(id); if (n && Array.isArray(n.c.invites)) out.push(...n.c.invites) }
return out
}
const sealedRoot = (root) => { const oc = (nodes.get(root) || {}).c || {}; return !!(oc.encryptedPayload || oc.encrypted === true) }
const selfJoinAllowed = (root, mm) => {
if (!sealedRoot(root)) return true
const invites = invitesOf(root)
if (invites.some(inv => inv && typeof inv === "object" && inv.public === true)) return true
if (mm.keyProof && lookupKeys(root).some(k => keyProofFor(k, mm.member) === mm.keyProof)) return true
const tribeChat = !!((nodes.get(root) || {}).c || {}).tribeId
return !!mm.code && invites.some(inv => tribeChat ? inviteMatches(inv, mm.code) : (typeof inv === "string" ? inv === mm.code : !!(inv && typeof inv === "object" && typeof inv.code === "string" && inv.code === mm.code)))
}
const consumedByRoot = new Map()
const codeClaimsByRoot = new Map()
for (const mm of memberMsgs) {
if (!nodes.has(mm.target)) continue
const r = rootOf(mm.target)
if (!memberByRoot.has(r)) memberByRoot.set(r, new Map())
const perMember = memberByRoot.get(r)
const prev = perMember.get(mm.member)
if (!prev || mm.ts >= prev.ts) perMember.set(mm.member, { on: mm.on, ts: mm.ts })
const ownerAuthor = nodes.get(r) && nodes.get(r).author
const self = mm.member === mm.author
if (!self && mm.author !== ownerAuthor) continue
if (mm.on && mm.code) {
if (!codeClaimsByRoot.has(r)) codeClaimsByRoot.set(r, [])
codeClaimsByRoot.get(r).push({ code: mm.code, member: mm.member, keyProof: mm.keyProof, byOwner: mm.author === ownerAuthor })
}
if (self && mm.author !== ownerAuthor && mm.on && !selfJoinAllowed(r, mm)) continue
if (mm.on && mm.code) {
if (!consumedByRoot.has(r)) consumedByRoot.set(r, new Set())
consumedByRoot.get(r).add(mm.code)
}
if (!memberByRoot.has(r)) memberByRoot.set(r, new Map())
const perMember = memberByRoot.get(r)
const prev = perMember.get(mm.member)
if (!prev || mm.ts >= prev.ts) perMember.set(mm.member, { on: mm.on, ts: mm.ts })
}
const isCodeConsumed = (rootId, code) => !!(code && consumedByRoot.has(rootId) && consumedByRoot.get(rootId).has(code))
const codeClaims = (rootId, codes) => (codeClaimsByRoot.get(rootId) || []).filter(c => codes.includes(c.code))
const rawRootOf = (id) => { let cur = id, g = 0; while (parent.has(cur) && g++ < 100000) cur = parent.get(cur); return cur }
return { tomb, nodes, parent: strictParent, child: strictChild, rawParent: parent, rawChild: child, rawRootOf, rootOf, tipOf, tipByRoot, msgNodes, memberByRoot, consumedByRoot, isCodeConsumed }
for (const [f, fNode] of nodes) {
const r = rootOf(f)
if (!nodes.has(r) || fNode.author === nodes.get(r).author) continue
const opened = decryptChatContent(fNode.c, r)
if (opened.undecryptable || !Array.isArray(opened.c && opened.c.members)) continue
if (!memberByRoot.has(r)) memberByRoot.set(r, new Map())
const perMember = memberByRoot.get(r)
const prev = perMember.get(fNode.author)
if (!prev || fNode.ts >= prev.ts) perMember.set(fNode.author, { on: opened.c.members.includes(fNode.author), ts: fNode.ts })
}
const reactionByKey = new Map()
for (const r of reactMsgs) {
const rk = `${r.target}|${r.author}|${r.emoji}`
const prev = reactionByKey.get(rk)
if (!prev || r.ts >= prev.ts) reactionByKey.set(rk, r)
}
const pinByTarget = new Map()
for (const pm of pinMsgs) {
const prev = pinByTarget.get(pm.target)
if (!prev || pm.ts >= prev.ts) pinByTarget.set(pm.target, pm)
}
return { tomb, nodes, rawRootOf, rootOf, tipOf, tipByRoot, msgNodes, memberByRoot, consumedByRoot, isCodeConsumed, codeClaims, reactionByKey, pinByTarget }
}
const resolveKeyChainSets = (chatRootId) => {
@ -248,13 +326,13 @@ module.exports = ({ cooler, tribeCrypto, chatCrypto, tribesModel }) => {
const { c, undecryptable } = decryptChatContent(rawC, rootId)
let ownerContent = c
let ownerAuthor = c.author || node.author
let ownerAuthor = node.author
if (idx) {
const ownerNode = idx.nodes.get(rootId)
if (ownerNode) {
const decOwner = decryptChatContent(ownerNode.c || {}, rootId)
ownerContent = decOwner.c
ownerAuthor = decOwner.c.author || ownerNode.author
ownerAuthor = ownerNode.author
}
}
const memberToggles = idx && idx.memberByRoot ? idx.memberByRoot.get(rootId) : null
@ -275,7 +353,7 @@ module.exports = ({ cooler, tribeCrypto, chatCrypto, tribesModel }) => {
tags: safeArr(c.tags),
members,
invites,
author: c.author || node.author,
author: node.author,
createdAt: c.createdAt || new Date(node.ts).toISOString(),
updatedAt: c.updatedAt || null,
encrypted: !!c.encrypted,
@ -304,8 +382,9 @@ module.exports = ({ cooler, tribeCrypto, chatCrypto, tribesModel }) => {
chatId: c.chatId || "",
text,
image: c.image || null,
replyTo: c.replyTo || null,
author: c.author || node.author,
mimeType: typeof c.mimeType === "string" ? c.mimeType : null,
replyTo: typeof c.replyTo === "string" ? c.replyTo : null,
author: node.author,
createdAt: c.createdAt || new Date(node.ts).toISOString()
}
}
@ -317,28 +396,27 @@ module.exports = ({ cooler, tribeCrypto, chatCrypto, tribesModel }) => {
})
const ensureMemberKeys = async (ssbClient, messages, chats) => {
if (!tribeCrypto) return
const distributed = new Map()
if (!tribeCrypto || !ownCrypto) return
const leavesByRoot = new Map()
for (const m of messages) {
const c = m.value && m.value.content
if (!c || c.type !== "tribe-keys" || !c.tribeId) continue
const mk = c.memberKeys
if (!mk || typeof mk !== "object") continue
if (!distributed.has(c.tribeId)) distributed.set(c.tribeId, new Set())
for (const id of Object.keys(mk)) distributed.get(c.tribeId).add(id)
if (!c || c.type !== "chatMember" || c.on !== false || !c.target || !c.member) continue
if (!leavesByRoot.has(c.target)) leavesByRoot.set(c.target, [])
leavesByRoot.get(c.target).push({ member: c.member, author: m.value.author, ts: Number(m.timestamp || m.value.timestamp || 0) })
}
const ssbKeys = require("../server/node_modules/ssb-keys")
for (const chat of safeArr(chats)) {
if (!chat || chat.undecryptable) continue
if (!chat || chat.undecryptable || chat.author !== ssbClient.id) continue
const rootId = chat.rootId
if (!rootId) continue
if (!rootId || !lookupKey(rootId)) continue
const members = safeArr(chat.members)
const leaves = (leavesByRoot.get(rootId) || []).filter(l => l.author === l.member || l.author === chat.author)
const plan = ownCrypto.keyPlan({ rootId, ownerId: chat.author, gen: lookupGen(rootId), members, messages, leaves })
if (plan.rotate) { await rotateChatKey(rootId, members); continue }
if (!plan.missing.length) continue
const key = lookupKey(rootId)
if (!key) continue
const have = distributed.get(rootId) || new Set()
const missing = safeArr(chat.members).filter(m => m && m !== ssbClient.id && !have.has(m))
if (!missing.length) continue
const memberKeys = {}
for (const m of missing) {
for (const m of plan.missing) {
try { memberKeys[m] = tribeCrypto.boxKeyForMember(key, m, ssbKeys) } catch (_) {}
}
if (!Object.keys(memberKeys).length) continue
@ -352,9 +430,22 @@ module.exports = ({ cooler, tribeCrypto, chatCrypto, tribesModel }) => {
new Promise((resolve, reject) => {
const content = { type: "chatMember", target: rootId, member, on: !!on, createdAt: new Date().toISOString() }
if (typeof code === "string" && code) content.code = code
const proofKey = on && member === ssbClient.id ? lookupKey(rootId) : null
if (proofKey) content.keyProof = keyProofFor(proofKey, member)
ssbClient.publish(content, (e, res) => e ? reject(e) : resolve(res))
})
const decryptTribeChats = async (idx) => {
if (!tribeCrypto || !tribesModel || typeof tribeCrypto.decryptFromTribe !== "function") return idx
for (const [k, n] of idx.nodes) {
const c = n.c || {}
if (!c.encryptedPayload || !c.tribeId) continue
const dec = await tribeCrypto.decryptFromTribe(c, tribesModel).catch(() => null)
if (dec && !dec._undecryptable) idx.nodes.set(k, { ...n, c: dec })
}
return idx
}
const collectChats = (idx) => {
const out = []
for (const [rootId, tipId] of idx.tipByRoot.entries()) {
@ -383,11 +474,8 @@ module.exports = ({ cooler, tribeCrypto, chatCrypto, tribesModel }) => {
const ssbClient = await openSsb()
const messages = await readAll(ssbClient)
const idx = buildIndex(messages)
let tip = id
while (idx.child.has(tip)) tip = idx.child.get(tip)
if (idx.tomb.has(tip)) throw new Error("Not found")
let root = tip
while (idx.parent.has(root)) root = idx.parent.get(root)
const root = idx.rootOf(id)
if (idx.tomb.has(idx.tipOf(root))) throw new Error("Not found")
return root
},
@ -395,13 +483,12 @@ module.exports = ({ cooler, tribeCrypto, chatCrypto, tribesModel }) => {
const ssbClient = await openSsb()
const messages = await readAll(ssbClient)
const idx = buildIndex(messages)
let tip = id
while (idx.child.has(tip)) tip = idx.child.get(tip)
const tip = idx.tipOf(idx.rootOf(id))
if (idx.tomb.has(tip)) throw new Error("Not found")
return tip
},
async createChat(title, description, image, category, status, tagsRaw, tribeId) {
async createChat(title, description, image, category, status, tagsRaw, tribeId, extra = {}) {
const ssbClient = await openSsb()
const userId = ssbClient.id
const blobId = image ? String(image).trim() || null : null
@ -422,7 +509,8 @@ module.exports = ({ cooler, tribeCrypto, chatCrypto, tribesModel }) => {
author: userId,
createdAt: now,
updatedAt: now,
...(tribeId ? { tribeId } : {})
...(tribeId ? { tribeId } : {}),
...(extra && typeof extra === "object" && typeof extra.courseId === "string" && extra.courseId ? { courseId: extra.courseId } : {})
}
if (!tribeCrypto) {
@ -478,13 +566,11 @@ module.exports = ({ cooler, tribeCrypto, chatCrypto, tribesModel }) => {
ssbClient.get(tipId, (err, item) => err || !item?.content ? reject(new Error("Chat not found")) : resolve(item))
})
const c = item.content
const rawAuthor = c.author || (c.encryptedPayload ? null : undefined)
if (rawAuthor && rawAuthor !== userId) throw new Error("Not the author")
if (item.author !== userId) throw new Error("Not the author")
const messages = await readAll(ssbClient)
const idx = buildIndex(messages)
let rootId = tipId
while (idx.parent.has(rootId)) rootId = idx.parent.get(rootId)
const rootId = idx.rootOf(tipId)
const node = { key: tipId, c, author: item.author, ts: item.timestamp || 0 }
const chat = buildChat(node, rootId, idx)
if (!chat) throw new Error("Invalid chat")
@ -544,7 +630,7 @@ module.exports = ({ cooler, tribeCrypto, chatCrypto, tribesModel }) => {
return new Promise((resolve, reject) => {
ssbClient.get(tipId, (err, item) => {
if (err || !item?.content) return reject(new Error("Chat not found"))
if (item.content.author && item.content.author !== userId) return reject(new Error("Not the author"))
if (item.author !== userId) return reject(new Error("Not the author"))
ssbClient.publish({ type: "tombstone", target: tipId, deletedAt: new Date().toISOString(), author: userId }, (e) => e ? reject(e) : resolve())
})
})
@ -555,11 +641,9 @@ module.exports = ({ cooler, tribeCrypto, chatCrypto, tribesModel }) => {
const userId = ssbClient.id
const messages = await readAll(ssbClient)
const idx = buildIndex(messages)
let tip = id
while (idx.child.has(tip)) tip = idx.child.get(tip)
const root = idx.rootOf(id)
const tip = idx.tipOf(root)
if (idx.tomb.has(tip)) throw new Error("Not found")
let root = tip
while (idx.parent.has(root)) root = idx.parent.get(root)
const node = idx.nodes.get(tip)
if (!node) throw new Error("Not found")
@ -592,18 +676,15 @@ module.exports = ({ cooler, tribeCrypto, chatCrypto, tribesModel }) => {
async getChatById(id) {
const ssbClient = await openSsb()
const messages = await readAll(ssbClient)
const idx = buildIndex(messages)
const idx = await decryptTribeChats(memoIndex('chats', messages, buildIndex))
let tip = id
while (idx.child.has(tip)) tip = idx.child.get(tip)
const root = idx.rootOf(id)
const tip = idx.tipOf(root)
if (idx.tomb.has(tip)) return null
const node = idx.nodes.get(tip)
if (!node || node.c.type !== "chat") return null
let root = tip
while (idx.parent.has(root)) root = idx.parent.get(root)
const chat = buildChat(node, root, idx)
if (!chat) return null
return chatCollab.fold(chat, collectChats(idx))
@ -613,7 +694,7 @@ module.exports = ({ cooler, tribeCrypto, chatCrypto, tribesModel }) => {
const ssbClient = await openSsb()
const uid = viewerId || ssbClient.id
const messages = await readAll(ssbClient)
const idx = buildIndex(messages)
const idx = await decryptTribeChats(buildIndex(messages))
const now = Date.now()
let list = chatCollab.visibleThenCollapsed(collectChats(idx), uid)
@ -631,7 +712,7 @@ module.exports = ({ cooler, tribeCrypto, chatCrypto, tribesModel }) => {
list = list.map(c => ({ ...c, lastMsgAt: lastMsgAt.get(c.rootId) || 0, lastMineAt: lastMineAt.get(c.rootId) || 0 }))
if (filter === "mine") list = list.filter(c => c.author === uid)
else if (filter === "recent") list = list.filter(c => new Date(c.createdAt).getTime() >= now - 86400000)
else if (filter === "recent") list = list.filter(c => Math.max(new Date(c.createdAt).getTime() || 0, Number(c.lastMsgAt || 0)) >= now - 86400000)
else if (filter === "open") list = list.filter(c => c.status === "OPEN" || c.status === "INVITE-ONLY")
else if (filter === "closed") list = list.filter(c => c.status === "CLOSED")
@ -665,13 +746,16 @@ module.exports = ({ cooler, tribeCrypto, chatCrypto, tribesModel }) => {
if (tribeCrypto) {
const inviteSalt = tribeCrypto.generateInviteSalt()
const ekChain = tribeCrypto.encryptChainForInvite([chat.rootId], code, inviteSalt)
const ident = opts.public ? { code, ...pubFlag } : { ch: hashCode(code, inviteSalt) }
if (ekChain) {
invite = { code, ekChain, salt: inviteSalt, gen: lookupGen(chat.rootId), ...pubFlag }
invite = { ...ident, ekChain, salt: inviteSalt, gen: lookupGen(chat.rootId) }
} else {
const chatKey = lookupKey(chat.rootId)
if (chatKey) {
const ek = tribeCrypto.encryptForInvite(chatKey, code, inviteSalt)
invite = { code, ek, salt: inviteSalt, gen: lookupGen(chat.rootId), ...pubFlag }
invite = { ...ident, ek, salt: inviteSalt, gen: lookupGen(chat.rootId) }
} else {
invite = { ...ident, salt: inviteSalt }
}
}
}
@ -702,6 +786,7 @@ module.exports = ({ cooler, tribeCrypto, chatCrypto, tribesModel }) => {
if (chat.author !== userId) throw new Error("Only the author can remove invites")
const invites = (Array.isArray(chat.invites) ? chat.invites : []).filter(inv => !(typeof inv === "object" && inv.public === true))
await this.updateChatById(chatId, { invites, members: chat.members, status: chat.status, title: chat.title, description: chat.description, image: chat.image, category: chat.category, tags: chat.tags })
if (!chat.tribeId) await rotateChatKey(chat.rootId, chat.members)
},
async joinByInvite(code) {
@ -721,12 +806,7 @@ module.exports = ({ cooler, tribeCrypto, chatCrypto, tribesModel }) => {
if (!chat || !chat.invites.length) continue
for (const inv of chat.invites) {
if (typeof inv === "string" && inv === code) {
matchedChat = chat; matchedInvite = inv; break
}
if (typeof inv === "object" && inv.code === code) {
matchedChat = chat; matchedInvite = inv; break
}
if (inviteMatches(inv, code)) { matchedChat = chat; matchedInvite = inv; break }
}
if (matchedChat) break
}
@ -735,29 +815,40 @@ module.exports = ({ cooler, tribeCrypto, chatCrypto, tribesModel }) => {
if (matchedChat.members.includes(userId)) throw new Error("Already a participant")
const isPublic = typeof matchedInvite === "object" && matchedInvite.public === true
if (!isPublic && idx.isCodeConsumed(matchedChat.rootId, code)) throw new Error("Invite already used")
const token = inviteToken(matchedInvite) || code
let chatKey = null
let chain = null
if (tribeCrypto && typeof matchedInvite === "object") {
if (matchedInvite.ekChain) {
const chain = tribeCrypto.decryptChainFromInvite(matchedInvite.ekChain, code, matchedInvite.salt, 3)
if (Array.isArray(chain) && chain.length) {
for (const entry of chain) {
if (Array.isArray(entry.keys) && entry.keys.length) {
tribeCrypto.setKeys(entry.rootId, entry.keys, entry.gen || entry.keys.length)
} else if (entry.key) {
tribeCrypto.setKey(entry.rootId, entry.key, entry.gen || 1)
}
}
chatKey = chain[0].key
}
chain = tribeCrypto.decryptChainFromInvite(matchedInvite.ekChain, code, matchedInvite.salt, 3)
if (Array.isArray(chain) && chain.length) chatKey = chain[0].key
else chain = null
} else if (matchedInvite.ek) {
chatKey = tribeCrypto.decryptFromInvite(matchedInvite.ek, code, matchedInvite.salt)
ownCrypto.setKey(matchedChat.rootId, chatKey, matchedInvite.gen || 1)
}
}
await publishMemberToggle(ssbClient, matchedChat.rootId, userId, true, isPublic ? undefined : code)
if (!isPublic) {
if (idx.isCodeConsumed(matchedChat.rootId, token) || idx.isCodeConsumed(matchedChat.rootId, code)) throw new Error("Invite already used")
const claims = idx.codeClaims(matchedChat.rootId, [token, code]).filter(c => c.member !== userId)
const spent = chatKey ? claims.some(c => c.byOwner || keyProofFor(chatKey, c.member) === c.keyProof) : claims.length > 0
if (spent) throw new Error("Invite already used")
}
if (chain) {
for (const entry of chain) {
if (Array.isArray(entry.keys) && entry.keys.length) {
tribeCrypto.setKeys(entry.rootId, entry.keys, entry.gen || entry.keys.length)
} else if (entry.key) {
tribeCrypto.setKey(entry.rootId, entry.key, entry.gen || 1)
}
}
} else if (chatKey) {
ownCrypto.setKey(matchedChat.rootId, chatKey, matchedInvite.gen || 1)
}
await publishMemberToggle(ssbClient, matchedChat.rootId, userId, true, isPublic ? undefined : token)
if (tribeCrypto && chatKey) {
try {
@ -801,9 +892,7 @@ module.exports = ({ cooler, tribeCrypto, chatCrypto, tribesModel }) => {
const chat = await this.getChatById(chatId)
if (!chat) throw new Error("Chat not found")
if (chat.author === userId) throw new Error("Author cannot leave their own chat")
const members = chat.members.filter(m => m !== userId)
await publishMemberToggle(ssbClient, chat.rootId, userId, false)
try { await rotateChatKey(chat.rootId, members) } catch (_) {}
},
async ingestKeys() { await ingestOwnTribeKeys() },
@ -825,7 +914,7 @@ module.exports = ({ cooler, tribeCrypto, chatCrypto, tribesModel }) => {
} catch (_) { return 0 }
},
async sendMessage(chatId, text, image = null, replyTo = null) {
async sendMessage(chatId, text, image = null, replyTo = null, mimeType = null) {
const ssbClient = await openSsb()
const userId = ssbClient.id
const chat = await this.getChatById(chatId)
@ -857,7 +946,14 @@ module.exports = ({ cooler, tribeCrypto, chatCrypto, tribesModel }) => {
createdAt: now
}
if (image) content.image = image
if (replyTo && typeof replyTo === "string" && replyTo.startsWith("%")) content.replyTo = replyTo
if (image && mimeType) content.mimeType = String(mimeType)
if (replyTo) {
const idxAll = buildIndex(messages)
const target = idxAll.msgNodes.get(replyTo)
if (target && (target.c.chatId === chat.rootId || idxAll.rawRootOf(target.c.chatId) === idxAll.rawRootOf(chat.rootId))) {
content.replyTo = replyTo
}
}
const chatIsEncrypted = !!(chat.tribeId) || !!lookupKey(chat.rootId)
if (chatIsEncrypted && tribeCrypto) {
@ -896,10 +992,74 @@ module.exports = ({ cooler, tribeCrypto, chatCrypto, tribesModel }) => {
if (msg) result.push(msg)
}
const viewerId = ssbClient.id
let chatAuthor = null
if (chatNode) chatAuthor = chatNode.author
const reactsByTarget = new Map()
for (const r of idx.reactionByKey.values()) {
if (!r.on) continue
if (!reactsByTarget.has(r.target)) reactsByTarget.set(r.target, [])
reactsByTarget.get(r.target).push(r)
}
const byKey = new Map(result.map(m => [m.key, m]))
for (const m of result) {
const counts = {}
const mine = {}
for (const e of REACTION_EMOJIS) { counts[e] = 0; mine[e] = false }
for (const r of reactsByTarget.get(m.key) || []) {
counts[r.emoji] += 1
if (r.author === viewerId) mine[r.emoji] = true
}
m.reactions = { counts, mine }
const pin = idx.pinByTarget.get(m.key)
m.pinned = !!(pin && pin.on && chatAuthor && pin.author === chatAuthor)
const quoted = m.replyTo ? byKey.get(m.replyTo) : null
m.reply = quoted ? { author: quoted.author, text: String(quoted.text || "").slice(0, 120) } : null
}
result.sort((a, b) => new Date(a.createdAt) - new Date(b.createdAt))
return result
},
async toggleReaction(chatId, targetKey, emoji) {
if (!REACTION_EMOJIS.includes(emoji)) throw new Error("Invalid reaction")
const ssbClient = await openSsb()
const userId = ssbClient.id
const chat = await this.getChatById(chatId)
if (!chat) throw new Error("Chat not found")
if (chat.status === "CLOSED") throw new Error("Chat is closed")
if (!chat.members.includes(userId)) {
if (chat.status === "OPEN") await this.joinChat(chatId)
else throw new Error("Not a participant")
}
const messages = await readAll(ssbClient)
const idx = buildIndex(messages)
const target = idx.msgNodes.get(targetKey)
if (!target || (target.c.chatId !== chat.rootId && idx.rawRootOf(target.c.chatId) !== idx.rawRootOf(chat.rootId))) throw new Error("Message not found")
const prev = idx.reactionByKey.get(`${targetKey}|${userId}|${emoji}`)
const on = !(prev && prev.on)
return new Promise((resolve, reject) => {
ssbClient.publish({ type: "chatReaction", chatId: chat.rootId, target: targetKey, emoji, on, author: userId, createdAt: new Date().toISOString() }, (e, res) => e ? reject(e) : resolve(res))
})
},
async togglePin(chatId, targetKey) {
const ssbClient = await openSsb()
const userId = ssbClient.id
const chat = await this.getChatById(chatId)
if (!chat) throw new Error("Chat not found")
if (chat.author !== userId) throw new Error("Only the chat author can pin")
const messages = await readAll(ssbClient)
const idx = buildIndex(messages)
const target = idx.msgNodes.get(targetKey)
if (!target || (target.c.chatId !== chat.rootId && idx.rawRootOf(target.c.chatId) !== idx.rawRootOf(chat.rootId))) throw new Error("Message not found")
const prev = idx.pinByTarget.get(targetKey)
const on = !(prev && prev.on && prev.author === userId)
return new Promise((resolve, reject) => {
ssbClient.publish({ type: "chatPin", chatId: chat.rootId, target: targetKey, on, author: userId, createdAt: new Date().toISOString() }, (e, res) => e ? reject(e) : resolve(res))
})
},
async getParticipants(chatRootId) {
const chat = await this.getChatById(chatRootId)
if (!chat) return []

30
src/models/content_visibility.js Normal file
View file

@ -0,0 +1,30 @@
const up = (v) => String(v == null ? '' : v).toUpperCase();
const arr = (v) => (Array.isArray(v) ? v : []);
const isPrivateFlag = (v) => {
if (v === false) return true;
const s = String(v == null ? '' : v).toLowerCase();
return s === 'private' || s === 'false' || s === '0';
};
const isContentVisibleTo = (type, c, viewer, author = null) => {
if (!c || typeof c !== 'object') return false;
if (c.encrypted === true || c.encryptedPayload || c.encryptedQuestion || c.encryptedText) return false;
const owner = String(author || c.author || '') === String(viewer);
switch (type) {
case 'task': return !isPrivateFlag(c.isPublic) || owner || arr(c.assignees).includes(viewer);
case 'event': return !isPrivateFlag(c.isPublic) || owner || arr(c.attendees).includes(viewer);
case 'forum': return !(c.isPrivate === true || c.isPrivate === 'true' || c.isPrivate === 'on') || owner;
case 'job': return up(c.visibility) !== 'HIDDEN' || owner || arr(c.subscribers).includes(viewer);
case 'housing':
case 'market': return up(c.visibility) !== 'HIDDEN' || owner;
case 'shop': return up(c.visibility) !== 'CLOSED' || owner;
case 'schoolCourse': return up(c.visibility) !== 'INVITE' || owner || arr(c.students).includes(viewer) || arr(c.invited).includes(viewer);
case 'tribe': return c.isAnonymous === false || owner || arr(c.members).includes(viewer);
case 'poll': return !c.chatId && !c.tribeId;
case 'room': return !c.tribeId && c.encrypted !== true;
case 'curriculum': return up(c.visibility) !== 'HIDDEN' || owner;
default: return !c.tribeId;
}
};
module.exports = { isContentVisibleTo };

134
src/models/courts_model.js
View file

@ -2,6 +2,9 @@ const pull = require('../server/node_modules/pull-stream');
const moment = require('../server/node_modules/moment');
const { getConfig } = require('../configs/config-manager.js');
const { buildValidatedTombstoneSet } = require('./tombstone_validator');
const { readTyped } = require('./typed_log');
const { buildVoteResults } = require('../backend/vote_tally');
const { versionGroups } = require('../backend/dedupe');
const logLimit = getConfig().ssbLogStream?.limit || 1000;
const CASE_ANSWER_DAYS = 7;
@ -16,6 +19,13 @@ const EVIDENCE_FIELDS = ['text', 'link', 'imageUrl'];
const ANSWER_FIELDS = ['stance', 'text'];
const VERDICT_FIELDS = ['result', 'orders'];
const SETTLEMENT_FIELDS = ['terms'];
const COURTS_TYPES = [
'courtsCase', 'courtsMediators', 'courtsJudge', 'courtsVoteLink', 'courtsVisibility',
'courtsAnswer', 'courtsEvidence', 'courtsVerdict', 'courtsSettlementProposal',
'courtsSettlementAccepted', 'courtsSupport', 'courtsNomination', 'courtsNominationVote',
'courts-key', 'tombstone'
];
const VOTE_LOG_TYPES = ['votes', 'votesVote', 'tombstone'];
module.exports = ({ cooler, services = {}, tribeCrypto }) => {
let ssb;
@ -73,12 +83,7 @@ module.exports = ({ cooler, services = {}, tribeCrypto }) => {
async function readLog() {
const ssbClient = await openSsb();
return new Promise((resolve, reject) => {
pull(
ssbClient.createLogStream({ limit: logLimit }),
pull.collect((err, arr) => (err ? reject(err) : resolve(arr)))
);
});
return readTyped(ssbClient, COURTS_TYPES, { limit: logLimit });
}
async function listByType(type) {
@ -86,13 +91,19 @@ module.exports = ({ cooler, services = {}, tribeCrypto }) => {
const tomb = buildValidatedTombstoneSet(msgs);
const rep = new Map();
const map = new Map();
const authorOf = new Map();
for (const m of msgs) {
const c = m.value?.content || m.content;
if (c && c.type === type) authorOf.set(m.key || m.id, m.value?.author);
}
for (const m of msgs) {
const k = m.key || m.id;
const c = m.value?.content || m.content;
if (!c) continue;
if (c.type === type) {
if (c.replaces && authorOf.has(c.replaces) && authorOf.get(c.replaces) !== m.value?.author) continue;
if (c.replaces) rep.set(c.replaces, k);
map.set(k, { id: k, ...c });
map.set(k, { ...c, id: k });
}
}
for (const oldId of rep.keys()) map.delete(oldId);
@ -252,7 +263,7 @@ if (c.type === type) {
myPublicPreference = c.publicPrefRespondent;
}
const rootId = c.rootCaseId || c.id;
const tally = c.voteId ? await voteTally(c.voteId) : null;
const tally = c.voteId ? await voteTally(c.voteId, [c.accuser, c.respondentId]) : null;
rows.push({
...c,
respondent: c.respondentId || c.respondent,
@ -284,27 +295,24 @@ if (c.type === type) {
for (const m of msgs) {
const c = m.value?.content || m.content;
if (!c || c.type !== 'courtsCase') continue;
nodes.set(m.key || m.id, { key: m.key || m.id, author: m.value?.author, c });
nodes.set(m.key || m.id, { key: m.key || m.id, author: m.value?.author, seq: m.value?.sequence || 0, ts: m.value?.timestamp || m.timestamp || 0, c });
}
const strictNext = new Map(), strictPrev = new Map();
for (const [k, n] of nodes) {
const t = n.c.replaces;
if (t) { const o = nodes.get(t); if (o && o.author === n.author) { strictNext.set(t, k); strictPrev.set(k, t); } }
}
return { nodes, strictNext, strictPrev };
const vg = versionGroups([...nodes.values()].map(n => ({ key: n.key, author: n.author, seq: n.seq, ts: n.ts, replaces: typeof n.c.replaces === 'string' ? n.c.replaces : null, claimedAuthor: typeof n.c.accuser === 'string' ? n.c.accuser : null })));
return { nodes, vg };
}
function strictCaseBase(msgs, id, idxIn) {
const idx = idxIn || collectCaseNodes(msgs);
const { nodes, strictNext, strictPrev } = idx;
const { nodes, vg } = idx;
const tomb = buildValidatedTombstoneSet(msgs);
let startKey = nodes.has(id) ? id : null;
if (!startKey) {
for (const [k, n] of nodes) { if (String(n.c.rootCaseId || '') === id) { startKey = k; break; } }
}
if (!startKey) return null;
let root = startKey, g = 0; while (strictPrev.has(root) && g++ < 100000) root = strictPrev.get(root);
let tip = root; g = 0; while (strictNext.has(tip) && g++ < 100000) tip = strictNext.get(tip);
const root = vg.rootOf(startKey);
if (vg.isForeignCopy(root)) return null;
const tip = vg.tipOf(root);
if (tomb.has(tip) || tomb.has(root)) return null;
const node = nodes.get(tip);
if (!node) return null;
@ -329,6 +337,7 @@ if (c.type === type) {
let medATs = 0, medRTs = 0, judgeId = '', judgeTs = 0, voteId = String(base.voteId || ''), voteTs = 0;
let prefA = base.publicPrefAccuser, prefATs = 0, prefR = base.publicPrefRespondent, prefRTs = 0;
let answered = false, settled = false, verdict = null, verdictTs = 0, verdictAt = null, settledAt = null;
const verdictClaims = [];
for (const m of msgs) {
const c = m.value?.content || m.content; if (!c) continue;
@ -351,20 +360,20 @@ if (c.type === type) {
answered = true;
} else if (c.type === 'courtsSettlementAccepted' && matchC(c) && isParty(author)) {
settled = true; settledAt = c.createdAt || new Date(ts).toISOString();
} else if (c.type === 'courtsVerdict' && matchC(c) && ts >= verdictTs) {
verdict = author; verdictTs = ts; verdictAt = c.createdAt || new Date(ts).toISOString();
} else if (c.type === 'courtsVerdict' && matchC(c)) {
verdictClaims.push({ author, ts, at: c.createdAt || new Date(ts).toISOString() });
}
}
let verdictValid = false;
if (verdict) {
if (method === 'JUDGE') verdictValid = (verdict === judgeId);
else if (method === 'MEDIATION') verdictValid = (medA.includes(verdict) || medR.includes(verdict));
else if (method === 'DICTATOR') verdictValid = (verdict === dictatorId);
const legit = await verdictAuthors({ method, judgeId, medA, medR, dictatorId, openedAt: base.openedAt || base.createdAt, accuser, respondentId });
for (const v of verdictClaims) {
if (!legit.has(v.author) || v.ts < verdictTs) continue;
verdict = v.author; verdictTs = v.ts; verdictAt = v.at;
}
const verdictValid = !!verdict;
let voteDecided = false;
if ((method === 'POPULAR' || method === 'KARMATOCRACY') && voteId) {
try { const t = await voteTally(voteId); if (t && t.closed && t.total > 0) voteDecided = true; } catch (_) {}
try { const t = await voteTally(voteId, [accuser, respondentId]); if (t && t.closed && t.total > 0 && (t.yes >= t.needed || t.no >= t.needed)) voteDecided = true; } catch (_) {}
}
base.mediatorsAccuser = medA;
@ -380,6 +389,33 @@ if (c.type === type) {
return base;
}
async function isRulingDictator(feedId, at) {
if (!feedId || !services.parliament || !services.parliament.listTerms) return false;
const atMs = new Date(at).getTime();
if (!Number.isFinite(atMs)) return false;
try {
const terms = await services.parliament.listTerms('all');
return (terms || []).some(t =>
String(t.method || '').toUpperCase() === 'DICTATORSHIP' &&
t.powerType === 'inhabitant' &&
String(t.powerId || '') === String(feedId) &&
new Date(t.startAt).getTime() <= atMs &&
atMs < new Date(t.endAt).getTime()
);
} catch (_) { return false; }
}
async function verdictAuthors({ method, judgeId, medA, medR, dictatorId, openedAt, accuser, respondentId }) {
const out = new Set();
if (method === 'JUDGE' && judgeId) out.add(String(judgeId));
else if (method === 'MEDIATION') for (const x of [...ensureArray(medA), ...ensureArray(medR)]) out.add(String(x));
else if (method === 'DICTATOR' && dictatorId && await isRulingDictator(dictatorId, openedAt)) out.add(String(dictatorId));
out.delete(String(accuser || ''));
out.delete(String(respondentId || ''));
out.delete('');
return out;
}
async function getCaseById(caseId) {
const id = String(caseId || '').trim();
if (!id) return null;
@ -649,17 +685,22 @@ if (c.type === type) {
return map;
}
async function voteTally(voteId) {
if (!voteId || !services.votes || !services.votes.getVoteById) return null;
async function voteTally(voteId, parties) {
if (!voteId) return null;
try {
const v = await services.votes.getVoteById(voteId);
const vm = v && v.votes ? v.votes : {};
const ssbClient = await openSsb();
const msgs = await readTyped(ssbClient, VOTE_LOG_TYPES, { limit: logLimit });
const r = buildVoteResults(msgs).get(voteId);
if (!r) return null;
const tomb = buildValidatedTombstoneSet(msgs);
if (tomb.has(r.rootId) || tomb.has(voteId)) return null;
if (Array.isArray(parties) && !parties.map(x => String(x || '')).includes(String(r.creator || ''))) return null;
const vm = r.votes || {};
const yes = Number(vm.YES ?? vm.Yes ?? vm.yes ?? 0);
const no = Number(vm.NO ?? vm.No ?? vm.no ?? 0);
const sum = Object.values(vm).reduce((s, n) => s + Number(n || 0), 0);
const total = Number(v.totalVotes ?? v.total ?? sum);
const deadline = v.deadline || v.endAt || v.expiresAt || null;
const closed = v.status === 'CLOSED' || (deadline && moment(deadline).isBefore(moment()));
const total = Number(r.totalVotes || 0);
const deadline = r.deadline || null;
const closed = !!(deadline && moment(deadline).isBefore(moment()));
const needed = Math.floor(total / 2) + 1;
return { yes, no, total, needed, closed, deadline };
} catch { return null; }
@ -868,8 +909,24 @@ if (c.type === type) {
.filter(matchCase)
.map(tryDecrypt)
.sort((a, b) => new Date(a.createdAt || 0) - new Date(b.createdAt || 0));
const legit = await verdictAuthors({
method: String(base.method || '').toUpperCase(),
judgeId,
medA: ma,
medR: mr,
dictatorId,
openedAt: base.openedAt || base.createdAt,
accuser: accuserId,
respondentId
});
const legitVerdictKeys = new Set();
for (const m of await readLog()) {
const c = m.value?.content;
if (c && c.type === 'courtsVerdict' && legit.has(String(m.value?.author || ''))) legitVerdictKeys.add(m.key || m.id);
}
const verdicts = verdictsAll
.filter(matchCase)
.filter(v => legitVerdictKeys.has(v.id))
.map(tryDecrypt)
.sort((a, b) => new Date(a.createdAt || 0) - new Date(b.createdAt || 0));
const verdict = verdicts.length ? verdicts[verdicts.length - 1] : null;
@ -884,7 +941,7 @@ if (c.type === type) {
const hasVerdict = !!verdict;
const supportsMap = await countSupportsByCase();
const supportCount = supportsMap.get(String(caseRootId)) || supportsMap.get(String(id)) || 0;
const tally = base.voteId ? await voteTally(base.voteId) : null;
const tally = base.voteId ? await voteTally(base.voteId, [accuserId, respondentId]) : null;
return {
...base,
id,
@ -919,12 +976,7 @@ if (c.type === type) {
const ssbKeys = require('../server/node_modules/ssb-keys');
const ssbConfig = require('../server/ssb_config');
const ssbClient = await openSsb();
const msgs = await new Promise((res, rej) => {
pull(
ssbClient.createLogStream({ limit: logLimit }),
pull.collect((err, arr) => (err ? rej(err) : res(arr)))
);
});
const msgs = await readLog();
for (const m of msgs) {
const c = m.value?.content;
if (!c || c.type !== 'courts-key') continue;

118
src/models/crypto.js
View file

@ -12,13 +12,17 @@ const SENSITIVE_FIELDS = [
const ENVELOPE_PRESERVE = new Set([
'type', 'tribeId', 'contentType', 'replaces', 'target', 'author',
'createdAt', 'updatedAt', 'encryptedPayload',
'mapId', 'calendarId', 'dateId', 'padId', 'roomId', 'parentId',
'mapId', 'calendarId', 'dateId', 'padId', 'roomId', 'parentId', 'courseId',
'members', 'invites', 'participants',
'_decrypted', '_undecryptable'
]);
const INVITE_SALT_LEGACY = 'SolarNET.HuB';
const INVITE_SCRYPT = { N: 131072, r: 8, p: 1, maxmem: 256 * 1024 * 1024 };
const INVITE_SCRYPT = { N: 16384, r: 8, p: 1, maxmem: 64 * 1024 * 1024 };
const INVITE_SCRYPT_LEGACY = { N: 131072, r: 8, p: 1, maxmem: 512 * 1024 * 1024 };
const INVITE_CACHE_MAX = 512;
const UNSAFE_KEYS = new Set(['__proto__', 'constructor', 'prototype']);
const INVITE_FAIL_TTL_MS = 10 * 60 * 1000;
const FP_INFO = Buffer.from('v1-fp', 'utf8');
const ENVELOPE_TYPE = 'tribe-msg';
@ -27,7 +31,7 @@ const KEY_DISTRIB_TYPE = 'tribe-keys-distrib';
const KEY_DISTRIB_BATCH = 7;
module.exports = (configPath, namespace = 'tribes') => {
const keysDir = path.join(configPath, 'keys');
const keysDir = require('../configs/state-manager').keysDir(configPath);
try { fs.mkdirSync(keysDir, { recursive: true, mode: 0o700 }); } catch (_) {}
const keyringPath = path.join(keysDir, `${namespace}-keys.json`);
@ -62,6 +66,8 @@ module.exports = (configPath, namespace = 'tribes') => {
};
const generateTribeKey = () => crypto.randomBytes(32).toString('hex');
const HEX_KEY = /^[0-9a-f]{64}$/i;
const validKey = (k) => typeof k === 'string' && HEX_KEY.test(k);
const getKey = (rid) => {
const e = keyring[rid];
@ -81,15 +87,17 @@ module.exports = (configPath, namespace = 'tribes') => {
const getAllRootIds = () => Object.keys(keyring);
const setKey = (rid, kHex, gen) => {
if (!validKey(kHex)) return;
keyring[rid] = { keys: [kHex], gen: gen || 1 };
saveKeyring();
};
const setKeys = (rid, ks, topGen) => {
if (!Array.isArray(ks) || !ks.length) return;
if (!Array.isArray(ks)) return;
const seen = new Set();
const dedup = [];
for (const k of ks) if (k && !seen.has(k)) { seen.add(k); dedup.push(k); }
for (const k of ks) if (validKey(k) && !seen.has(k)) { seen.add(k); dedup.push(k); }
if (!dedup.length) return;
keyring[rid] = { keys: dedup, gen: topGen || dedup.length };
saveKeyring();
};
@ -97,8 +105,9 @@ module.exports = (configPath, namespace = 'tribes') => {
const mergeKeys = (rid, incoming, topGen) => {
const e = keyring[rid] || { keys: [], gen: 0 };
const seen = new Set(e.keys);
const merged = [...e.keys];
for (const k of incoming) if (k && !seen.has(k)) { seen.add(k); merged.push(k); }
const fresh = [];
for (const k of (Array.isArray(incoming) ? incoming : [])) if (validKey(k) && !seen.has(k)) { seen.add(k); fresh.push(k); }
const merged = [...fresh, ...e.keys];
keyring[rid] = { keys: merged, gen: Math.max(e.gen || 0, topGen || merged.length) };
saveKeyring();
return keyring[rid].gen;
@ -106,6 +115,7 @@ module.exports = (configPath, namespace = 'tribes') => {
const addNewKey = (rid, kHex) => {
const e = keyring[rid] || { keys: [], gen: 0 };
if (!validKey(kHex)) return e.gen;
if (e.keys.includes(kHex)) return e.gen;
e.keys.unshift(kHex);
e.gen = (e.gen || 0) + 1;
@ -114,6 +124,27 @@ module.exports = (configPath, namespace = 'tribes') => {
return e.gen;
};
const keyPlan = ({ rootId, ownerId, gen, members, messages, leaves }) => {
const stamp = (m) => Number(m.timestamp || (m.value && m.value.timestamp) || 0);
const g = Number(gen) || 0;
let genStart = g > 1 ? null : 0;
const lastGot = new Map();
for (const m of (Array.isArray(messages) ? messages : [])) {
const c = m && m.value && m.value.content;
if (!c || c.type !== 'tribe-keys' || c.tribeId !== rootId || m.value.author !== ownerId) continue;
if (!c.memberKeys || typeof c.memberKeys !== 'object') continue;
const t = stamp(m);
for (const id of Object.keys(c.memberKeys)) lastGot.set(id, Math.max(lastGot.get(id) || 0, t));
if (g > 1 && Number(c.generation) === g) genStart = genStart === null ? t : Math.min(genStart, t);
}
if (genStart === null) genStart = 0;
const current = new Set((Array.isArray(members) ? members : []).filter(Boolean));
const gone = (Array.isArray(leaves) ? leaves : []).filter((l) => l && l.member && l.member !== ownerId);
const rotate = g > 0 && gone.some((l) => !current.has(l.member) && l.ts > genStart);
const missing = rotate ? [] : [...current].filter((id) => id !== ownerId && (!lastGot.has(id) || gone.some((l) => l.member === id && l.ts > lastGot.get(id))));
return { rotate, missing };
};
const dropKey = (rid) => {
if (keyring[rid]) {
delete keyring[rid];
@ -121,8 +152,15 @@ module.exports = (configPath, namespace = 'tribes') => {
}
};
const fingerprint = (kHex) =>
crypto.createHmac('sha256', Buffer.from(kHex, 'hex')).update(FP_INFO).digest('hex').slice(0, 32);
const fingerprints = new Map();
const fingerprint = (kHex) => {
let fp = fingerprints.get(kHex);
if (!fp) {
fp = crypto.createHmac('sha256', Buffer.from(kHex, 'hex')).update(FP_INFO).digest('hex').slice(0, 32);
fingerprints.set(kHex, fp);
}
return fp;
};
const buildFingerprintIndex = () => {
const m = new Map();
@ -200,9 +238,9 @@ module.exports = (configPath, namespace = 'tribes') => {
const generateInviteSalt = () => crypto.randomBytes(16).toString('hex');
const deriveInviteKey = (code, salt) => {
const deriveInviteKey = (code, salt, params = INVITE_SCRYPT) => {
const s = (salt === undefined || salt === null || salt === '') ? INVITE_SALT_LEGACY : salt;
return crypto.scryptSync(code, s, 32, INVITE_SCRYPT);
return crypto.scryptSync(code, s, 32, params);
};
const hashInviteCode = (code, salt) => {
@ -218,9 +256,44 @@ module.exports = (configPath, namespace = 'tribes') => {
return encryptWithKey(tribeKeyHex, derived.toString('hex'), inviteAad(inviteCode, salt));
};
const derivedCache = new Map();
const failedCache = new Map();
const remember = (map, k, v, max) => {
map.delete(k);
map.set(k, v);
while (map.size > max) map.delete(map.keys().next().value);
};
const derivedKeyName = (code, salt, params) => `${params.N}|${salt == null ? '' : salt}|${code}`;
const openWithDerived = (payload, code, salt, params, aad) => {
const name = derivedKeyName(code, salt, params);
const cached = derivedCache.get(name);
const derived = cached || deriveInviteKey(code, salt, params);
const out = decryptWithKey(payload, derived.toString('hex'), aad);
if (out && !cached) remember(derivedCache, name, derived, INVITE_CACHE_MAX);
return out;
};
const failureName = (payload, code, salt) => crypto.createHash('sha256').update(`${String(payload)}|${code}|${salt == null ? '' : salt}`).digest('hex');
const recentlyFailed = (name) => {
const at = failedCache.get(name);
return !!at && Date.now() - at < INVITE_FAIL_TTL_MS;
};
const decryptFromInvite = (encryptedKey, inviteCode, salt) => {
const derived = deriveInviteKey(inviteCode, salt);
return decryptWithKey(encryptedKey, derived.toString('hex'), inviteAad(inviteCode, salt));
const failure = failureName(encryptedKey, inviteCode, salt);
if (recentlyFailed(failure)) throw new Error('Unsupported state or unable to authenticate data');
const aad = inviteAad(inviteCode, salt);
try {
const out = openWithDerived(encryptedKey, inviteCode, salt, INVITE_SCRYPT, aad);
if (out) return out;
} catch (_) {}
for (let attempt = 0; attempt < 3; attempt++) {
try {
const out = openWithDerived(encryptedKey, inviteCode, salt, INVITE_SCRYPT_LEGACY, aad);
if (out) return out;
} catch (_) {}
}
remember(failedCache, failure, Date.now(), INVITE_CACHE_MAX);
throw new Error('Unsupported state or unable to authenticate data');
};
const INVITE_ENCRYPT_ATTEMPTS = 3;
@ -244,9 +317,15 @@ module.exports = (configPath, namespace = 'tribes') => {
};
const decryptChainOnce = (encryptedPayload, code, salt) => {
const k = deriveInviteKey(code, salt);
let json = null
try {
json = openWithDerived(encryptedPayload, code, salt, INVITE_SCRYPT, inviteAad(code, salt));
} catch (_) {
try {
json = openWithDerived(encryptedPayload, code, salt, INVITE_SCRYPT_LEGACY, inviteAad(code, salt));
} catch (_) { return null; }
}
try {
const json = decryptWithKey(encryptedPayload, k.toString('hex'), inviteAad(code, salt));
const parsed = JSON.parse(json);
if (Array.isArray(parsed) && parsed.every(e => e && e.rootId && Array.isArray(e.keys) && e.keys.length)) {
return parsed.map(e => ({
@ -261,11 +340,14 @@ module.exports = (configPath, namespace = 'tribes') => {
};
const decryptChainFromInvite = (encryptedPayload, code, salt, attempts = 1) => {
const failure = failureName(encryptedPayload, code, salt);
if (recentlyFailed(failure)) return null;
const tries = Math.max(1, Number(attempts) || 1);
for (let i = 0; i < tries; i++) {
const chain = decryptChainOnce(encryptedPayload, code, salt);
if (chain) return chain;
}
remember(failedCache, failure, Date.now(), INVITE_CACHE_MAX);
return null;
};
@ -365,7 +447,9 @@ module.exports = (configPath, namespace = 'tribes') => {
const payload = JSON.parse(plaintext);
const result = { ...content };
delete result.encryptedPayload;
Object.assign(result, payload);
if (payload && typeof payload === 'object' && !Array.isArray(payload)) {
for (const [k, v] of Object.entries(payload)) if (!UNSAFE_KEYS.has(k) && !(k in envelope)) result[k] = v;
}
return result;
} catch (_) {}
}
@ -534,7 +618,7 @@ module.exports = (configPath, namespace = 'tribes') => {
ENVELOPE_TYPE, ENVELOPE_VERSION, KEY_DISTRIB_TYPE, KEY_DISTRIB_BATCH,
loadKeyring, saveKeyring,
generateTribeKey, getKey, getKeys, getGen, getAllRootIds,
setKey, setKeys, mergeKeys, addNewKey, dropKey,
setKey, setKeys, mergeKeys, addNewKey, dropKey, keyPlan,
fingerprint, buildFingerprintIndex,
isTribeMsg, wrapMsg, unwrapMsg,
encryptWithKey, decryptWithKey,

50
src/models/cv_model.js
View file

@ -1,8 +1,11 @@
const pull = require('../server/node_modules/pull-stream');
const { getConfig } = require('../configs/config-manager.js');
const { buildValidatedTombstoneSet } = require('./tombstone_validator');
const { readTyped } = require('./typed_log');
const logLimit = getConfig().ssbLogStream?.limit || 1000;
const CV_TYPES = ['curriculum', 'tombstone'];
const extractBlobId = str => {
if (!str || typeof str !== 'string') return null;
const match = str.match(/\(([^)]+\.sha256)\)/);
@ -24,7 +27,7 @@ module.exports = ({ cooler }) => {
return {
type: 'curriculum',
async createCV(data, photoBlobId) {
async createCV(data, photoBlobId, pdfBlobId = null) {
const ssbClient = await openSsb();
const userId = ssbClient.id;
const content = {
@ -33,6 +36,7 @@ module.exports = ({ cooler }) => {
name: data.name,
description: data.description,
photo: extractBlobId(photoBlobId) || null,
pdf: extractBlobId(pdfBlobId) || null,
contact: userId,
personalSkills: parseCSV(data.personalSkills),
personalExperiences: data.personalExperiences || '',
@ -56,7 +60,7 @@ module.exports = ({ cooler }) => {
});
},
async updateCV(id, data, photoBlobId) {
async updateCV(id, data, photoBlobId, pdfBlobId = null) {
const ssbClient = await openSsb();
const userId = ssbClient.id;
@ -88,6 +92,7 @@ module.exports = ({ cooler }) => {
name: data.name,
description: data.description,
photo: extractBlobId(photoBlobId) || old.content.photo || null,
pdf: extractBlobId(pdfBlobId) || old.content.pdf || null,
contact: userId,
personalSkills: parseCSV(data.personalSkills),
personalExperiences: data.personalExperiences || '',
@ -151,34 +156,27 @@ module.exports = ({ cooler }) => {
const userId = ssbClient.id;
const authorId = targetUserId || userId;
return new Promise((resolve, reject) => {
pull(
ssbClient.createLogStream({ limit: logLimit }),
pull.collect((err, msgs) => {
if (err) return reject(err);
const msgs = await readTyped(ssbClient, CV_TYPES, { limit: logLimit });
const tombstoned = buildValidatedTombstoneSet(msgs);
const tombstoned = buildValidatedTombstoneSet(msgs);
const cvMsgs = msgs
.filter(m =>
m.value?.content?.type === 'curriculum' &&
m.value.author === authorId &&
!tombstoned.has(m.key)
)
.sort((a, b) => b.value.timestamp - a.value.timestamp);
const cvMsgs = msgs
.filter(m =>
m.value?.content?.type === 'curriculum' &&
m.value.author === authorId &&
!tombstoned.has(m.key)
)
.sort((a, b) => b.value.timestamp - a.value.timestamp);
if (!cvMsgs.length) {
return resolve(null);
}
if (!cvMsgs.length) {
return null;
}
const latest = cvMsgs[0];
const c = latest.value.content;
const visibility = String(c.visibility || 'PUBLIC').toUpperCase() === 'HIDDEN' ? 'HIDDEN' : 'PUBLIC';
if (visibility === 'HIDDEN' && authorId !== userId) return resolve(null);
resolve({ id: latest.key, ...c, visibility });
})
);
});
const latest = cvMsgs[0];
const c = latest.value.content;
const visibility = String(c.visibility || 'PUBLIC').toUpperCase() === 'HIDDEN' ? 'HIDDEN' : 'PUBLIC';
if (visibility === 'HIDDEN' && authorId !== userId) return null;
return { id: latest.key, ...c, visibility };
}
};
};

274
src/models/data_model.js
View file

@ -1,6 +1,9 @@
const pull = require('../server/node_modules/pull-stream');
const { getConfig } = require('../configs/config-manager.js');
const { buildValidatedTombstoneSet } = require('./tombstone_validator');
const { readTyped } = require('./typed_log');
const { isContentVisibleTo } = require('./content_visibility');
const { versionGroups, versionEntry } = require('../backend/dedupe');
const logLimit = getConfig().ssbLogStream?.limit || 1000;
@ -24,11 +27,20 @@ const KINDS = {
documents: { type: 'document', href: (id) => `/documents/${encodeURIComponent(id)}` },
bookmarks: { type: 'bookmark', href: (id) => `/bookmarks/${encodeURIComponent(id)}` },
torrents: { type: 'torrent', href: (id) => `/torrents/${encodeURIComponent(id)}` },
files: { type: 'file', href: (id) => `/files/${encodeURIComponent(id)}` },
chats: { type: 'chat', href: (id) => `/chats/${encodeURIComponent(id)}` },
pads: { type: 'pad', href: (id) => `/pads/${encodeURIComponent(id)}` },
rooms: { type: 'room', href: (id) => `/rooms/${encodeURIComponent(id)}` },
maps: { type: 'map', href: (id) => `/maps/${encodeURIComponent(id)}` },
calendars: { type: 'calendar', href: (id) => `/calendars/${encodeURIComponent(id)}` },
forum: { type: 'forum', href: (id) => `/forum/${encodeURIComponent(id)}` }
forum: { type: 'forum', href: (id) => `/forum/${encodeURIComponent(id)}` },
school: { type: 'schoolCourse', href: (id) => `/school/course/${encodeURIComponent(id)}` },
wiki: { type: 'wikiPage', href: (id) => `/wiki/${encodeURIComponent(id)}` },
emergencies: { type: 'emergency', href: (id) => `/emergencies/${encodeURIComponent(id)}` },
mailing: { type: 'mailingList', href: (id) => `/mailing/${encodeURIComponent(id)}` },
logistics: { type: 'logisticsRoute', href: (id) => `/logistics/${encodeURIComponent(id)}` },
podcasts: { type: 'podcast', href: (id) => `/podcasts/${encodeURIComponent(id)}` },
campaigns: { type: 'campaign', href: (id) => `/campaigns/${encodeURIComponent(id)}` }
};
const KIND_BY_TYPE = Object.fromEntries(Object.entries(KINDS).map(([k, v]) => [v.type, k]));
@ -49,7 +61,7 @@ const TITLE_STOPWORDS = new Set([
const titleTerms = (c) => String(c.title || c.name || c.question || c.concept || '')
.toLowerCase()
.split(/[^0-9A-Za-zÀ-￿]+/) // sin \p{}: nodejs-mobile no lleva ICU completo
.split(/[^\w\u0080-\uFFFF]+/)
.filter(w => w.length >= 3 && !TITLE_STOPWORDS.has(w) && !/^\d+$/.test(w));
const termsOf = (kind, c) => {
@ -68,9 +80,30 @@ const termsOf = (kind, c) => {
return Array.from(new Set(out.map(norm).filter(t => t && !PLACEHOLDERS.has(t))));
};
const DESC_TERMS_MAX = 40;
const descTerms = (c) => String(c.description || c.text || c.body || '')
.toLowerCase()
.split(/[^\w\u0080-\uFFFF]+/)
.filter(w => w.length >= 4 && !TITLE_STOPWORDS.has(w) && !/^\d+$/.test(w))
.slice(0, DESC_TERMS_MAX);
const FAV_KIND = Object.fromEntries(Object.keys(KINDS).map(k => [k, k === 'votes' ? 'polls' : k]));
const ALIKE_MIN_AFFINITY = 0.3;
const RATED_MIN_OPINIONS = 3;
const REASON_ORDER = ['related', 'content', 'mutual', 'following', 'supportsYou', 'tribe', 'alike', 'cv', 'pinned', 'rated', 'near'];
const opinionsTotalOf = (c) => {
const ops = c && typeof c.opinions === 'object' && c.opinions ? c.opinions : {};
let total = 0;
for (const v of Object.values(ops)) total += Number(v) || 0;
return total;
};
const excerpt = (s) => String(s || '').replace(/!?\[[^\]]*\]\([^)]*\)/g, ' ').replace(/[#*_>`~|]/g, ' ').replace(/\s+/g, ' ').trim().slice(0, 90);
const titleOf = (kind, c, author) => {
if (kind === 'inhabitants') return c.name || author || '';
return c.title || c.name || c.question || c.concept || '';
if (kind === 'inhabitants') return c.name || '';
return c.title || c.name || c.question || c.concept || c.subject || excerpt(typeof c.text === 'string' ? c.text : '') || excerpt(typeof c.description === 'string' ? c.description : '') || '';
};
const jaccard = (a, b) => {
@ -80,17 +113,13 @@ const jaccard = (a, b) => {
return { score: union > 0 ? common.length / union : 0, common };
};
module.exports = ({ cooler }) => {
module.exports = ({ cooler, favoriteIdsFor = null }) => {
let ssb;
const openSsb = async () => { if (!ssb) ssb = await cooler.open(); return ssb; };
const getAllMessages = async (ssbClient) =>
new Promise((resolve, reject) => {
pull(
ssbClient.createLogStream({ limit: logLimit }),
pull.collect((err, msgs) => (err ? reject(err) : resolve(msgs)))
);
});
const DATA_TYPES = [...Object.values(KINDS).map(v => v.type), 'tombstone', 'contact', 'about'];
const getAllMessages = async (ssbClient) => readTyped(ssbClient, DATA_TYPES, { limit: logLimit });
const buildGraph = async () => {
const ssbClient = await openSsb();
@ -98,40 +127,74 @@ module.exports = ({ cooler }) => {
const messages = await getAllMessages(ssbClient);
const tomb = buildValidatedTombstoneSet(messages);
const latestByKey = new Map();
const replaced = new Set();
const candidates = new Map();
const versions = [];
const follows = new Map();
const followers = new Map();
const aboutTerms = new Set();
for (const m of messages) {
const v = m && m.value;
const c = v && v.content;
if (!c || typeof c !== 'object' || !c.type) continue;
if (tomb.has(m.key)) continue;
if (c.encryptedPayload || c.encryptedQuestion) continue;
if (c.type === 'contact') {
if (typeof c.contact !== 'string' || typeof c.following !== 'boolean') continue;
if (v.author === viewerId) follows.set(c.contact, c.following);
else if (c.contact === viewerId) followers.set(v.author, c.following);
continue;
}
if (c.type === 'about') {
if (v.author === viewerId && c.about === viewerId && typeof c.description === 'string') for (const t of descTerms(c)) aboutTerms.add(norm(t));
continue;
}
if (c.tribeId && c.type !== 'tribe') continue;
const kind = KIND_BY_TYPE[c.type];
if (!kind) continue;
if (typeof c.replaces === 'string') replaced.add(c.replaces);
latestByKey.set(m.key, { key: m.key, author: v.author, ts: v.timestamp || m.timestamp || 0, kind, c });
versions.push(versionEntry(m));
if (tomb.has(m.key)) continue;
if (!isContentVisibleTo(c.type, c, viewerId, v.author)) continue;
candidates.set(m.key, { key: m.key, author: v.author, ts: v.timestamp || m.timestamp || 0, kind, c });
}
const vg = versionGroups(versions);
const latestByRoot = new Map();
for (const root of vg.roots) {
const tip = candidates.get(vg.tipOf(root));
if (!tip) continue;
if (tip.kind !== 'tribes') { latestByRoot.set(root, { ...tip, key: root }); continue; }
const members = new Set(Array.isArray(tip.c.members) ? tip.c.members : []);
const lastByAuthor = new Map();
for (const k of vg.groups.get(root)) {
const n = candidates.get(k);
if (!n || n.author === tip.author || n.ts < tip.ts) continue;
const prev = lastByAuthor.get(n.author);
if (!prev || n.ts >= prev.ts) lastByAuthor.set(n.author, n);
}
for (const [a, n] of lastByAuthor) { if (Array.isArray(n.c.members) && n.c.members.includes(a)) members.add(a); else members.delete(a); }
latestByRoot.set(root, { ...tip, key: root, c: { ...tip.c, members: [...members] } });
}
const nodes = [];
const byAuthorCv = new Map();
for (const node of latestByKey.values()) {
if (replaced.has(node.key)) continue;
for (const node of latestByRoot.values()) {
const coreTerms = termsOf(node.kind, node.c);
const extra = node.kind === 'inhabitants' ? [] : titleTerms(node.c).map(norm).filter(t => t && !PLACEHOLDERS.has(t));
const extra = node.kind === 'inhabitants' ? [] : [...titleTerms(node.c), ...descTerms(node.c)].map(norm).filter(t => t && !PLACEHOLDERS.has(t));
const terms = Array.from(new Set([...coreTerms, ...extra]));
if (!terms.length) continue;
const entry = {
id: node.key,
kind: node.kind,
author: node.c.author || node.author,
title: titleOf(node.kind, node.c, node.c.author || node.author),
ctype: KINDS[node.kind] ? KINDS[node.kind].type : '',
author: node.author,
title: titleOf(node.kind, node.c, node.author),
terms,
termSet: new Set(terms),
coreTermSet: new Set(coreTerms),
ts: node.ts,
createdAt: node.c.createdAt || new Date(node.ts).toISOString(),
href: KINDS[node.kind].href(node.key, node.c)
href: KINDS[node.kind].href(node.key, node.c),
members: node.kind === 'tribes' && Array.isArray(node.c.members) ? node.c.members : null,
location: norm(node.c.location || ''),
opinionsTotal: opinionsTotalOf(node.c)
};
if (node.kind === 'inhabitants') {
const prev = byAuthorCv.get(entry.author);
@ -143,54 +206,168 @@ module.exports = ({ cooler }) => {
}
for (const cv of byAuthorCv.values()) nodes.push(cv);
return { viewerId, nodes, cvByAuthor: byAuthorCv };
const following = new Set([...follows.entries()].filter(([, on]) => on).map(([id]) => id));
const supporters = new Set([...followers.entries()].filter(([, on]) => on).map(([id]) => id));
return { viewerId, nodes, cvByAuthor: byAuthorCv, following, supporters, aboutTerms };
};
const favoriteIds = async () => {
if (typeof favoriteIdsFor !== 'function') return new Set();
const out = new Set();
await Promise.all(Object.values(FAV_KIND).map(async (k) => {
try { for (const id of await favoriteIdsFor(k)) out.add(String(id)); } catch (_) {}
}));
return out;
};
const diversify = (items) => {
const leaders = new Map();
for (const it of items) if (!leaders.has(it.kind)) leaders.set(it.kind, it);
const lead = new Set(leaders.values());
return [...leaders.values()].sort((x, y) => y.score - x.score || y.ts - x.ts)
.concat(items.filter(it => !lead.has(it)));
};
const MAX_ENTITIES = 400;
const MAX_PAIRS = 300;
const strip = (n) => ({
id: n.id, kind: n.kind, author: n.author, title: n.title,
id: n.id, kind: n.kind, ctype: n.ctype, author: n.author, title: n.title,
href: n.href, createdAt: n.createdAt, ts: n.ts
});
const buildProfile = async (graph) => {
const { viewerId, nodes, cvByAuthor, aboutTerms } = graph;
const use = nodes.slice(0, MAX_ENTITIES);
const favs = await favoriteIds();
const myTermSet = new Set();
const cvTerms = new Set();
const contentTerms = new Set();
const mineCv = cvByAuthor.get(viewerId);
if (mineCv) for (const t of mineCv.terms) { myTermSet.add(t); cvTerms.add(t); }
for (const t of aboutTerms) if (t && !PLACEHOLDERS.has(t)) myTermSet.add(t);
const tribeMates = new Set();
const pinnedTerms = new Set();
let myLocation = mineCv ? mineCv.location : '';
for (const n of use) {
if (String(n.author) === String(viewerId)) {
for (const t of n.terms) { myTermSet.add(t); contentTerms.add(t); }
if (!myLocation && n.location) myLocation = n.location;
}
if (favs.has(String(n.id))) for (const t of n.terms) { myTermSet.add(t); pinnedTerms.add(t); }
if (n.members && (String(n.author) === String(viewerId) || n.members.includes(viewerId))) for (const m of n.members) tribeMates.add(m);
}
tribeMates.delete(viewerId);
const df = new Map();
for (const n of use) for (const t of n.termSet) df.set(t, (df.get(t) || 0) + 1);
const total = use.length || 1;
const weightOf = (t) => Math.log(1 + total / (df.get(t) || 1));
return { use, myTermSet, cvTerms, contentTerms, pinnedTerms, tribeMates, myLocation, df, weightOf, total };
};
const affinitiesFor = (viewerId, { use, myTermSet, pinnedTerms, tribeMates, weightOf }) => {
const termsByAuthor = new Map();
for (const n of use) {
if (String(n.author) === String(viewerId)) continue;
if (!termsByAuthor.has(n.author)) termsByAuthor.set(n.author, new Set());
const set = termsByAuthor.get(n.author);
for (const t of n.termSet) set.add(t);
}
let myW = 0;
for (const t of myTermSet) myW += weightOf(t);
const byAuthor = new Map();
for (const [author, terms] of termsByAuthor) {
const common = [...terms].filter(t => myTermSet.has(t));
let commonW = 0;
for (const t of common) commonW += weightOf(t);
let theirW = 0;
for (const t of terms) theirW += weightOf(t);
const floor = Math.min(myW, theirW);
const score = floor > 0 ? Math.min(1, commonW / floor) : 0;
common.sort((x, y) => weightOf(y) - weightOf(x) || x.localeCompare(y));
const reasons = [];
if (tribeMates.has(String(author))) reasons.push('tribe');
if (common.some(t => pinnedTerms.has(t))) reasons.push('pinned');
byAuthor.set(author, { score, common, reasons });
}
for (const m of tribeMates) if (!byAuthor.has(m)) byAuthor.set(m, { score: 0, common: [], reasons: ['tribe'] });
return byAuthor;
};
return {
KINDS: Object.keys(KINDS),
async authorAffinities() {
const graph = await buildGraph();
const profile = await buildProfile(graph);
return { viewerId: graph.viewerId, byAuthor: affinitiesFor(graph.viewerId, profile), hasProfile: profile.myTermSet.size > 0 };
},
async listMatches(filter = 'ALL', opts = {}) {
const { viewerId, nodes, cvByAuthor } = await buildGraph();
const use = nodes.slice(0, MAX_ENTITIES);
const graph = await buildGraph();
const { viewerId, following, supporters } = graph;
const profile = await buildProfile(graph);
const { use, myTermSet, cvTerms, contentTerms, pinnedTerms, tribeMates, myLocation, df, weightOf, total } = profile;
const affinity = affinitiesFor(viewerId, profile);
const f = String(filter || 'ALL').toUpperCase();
const myTermSet = new Set();
const mineCv = cvByAuthor.get(viewerId);
if (mineCv) for (const t of mineCv.terms) myTermSet.add(t);
const expanded = new Set();
for (const n of use) {
if (String(n.author) === String(viewerId)) for (const t of n.terms) myTermSet.add(t);
if (String(n.author) === String(viewerId)) continue;
if (![...n.termSet].some(t => myTermSet.has(t))) continue;
for (const t of n.termSet) if (!myTermSet.has(t)) expanded.add(t);
}
const df = new Map();
for (const n of use) for (const t of n.termSet) df.set(t, (df.get(t) || 0) + 1);
const total = use.length || 1;
const weightOf = (t) => Math.log(1 + total / (df.get(t) || 1));
let out = [];
for (const n of use) {
if (!myTermSet.size) break;
if (String(n.author) === String(viewerId)) continue;
const common = [...n.termSet].filter(t => myTermSet.has(t));
if (!common.length) continue;
let commonW = 0;
for (const t of common) commonW += weightOf(t);
let indirectW = 0;
for (const t of n.termSet) if (!myTermSet.has(t) && expanded.has(t)) indirectW += weightOf(t);
let itemW = 0;
for (const t of n.termSet) itemW += weightOf(t);
const score = itemW > 0 ? Math.min(1, commonW / itemW) : 0;
if (score <= 0) continue;
const direct = itemW > 0 ? Math.min(1, commonW / itemW) : 0;
const indirect = itemW > 0 ? Math.min(1, indirectW / itemW) : 0;
let dfSum = 0;
for (const t of n.termSet) dfSum += (df.get(t) || 1) / total;
const connectivity = n.termSet.size ? dfSum / n.termSet.size : 0;
const base = 0.8 * direct + 0.15 * indirect + 0.05 * connectivity;
const ageDays = Math.max(0, (Date.now() - (n.ts || 0)) / 86400000);
const recency = 0.8 + 0.2 * Math.exp(-ageDays / 60);
const social = (following.has(String(n.author)) ? 0.1 : 0) + (tribeMates.has(String(n.author)) ? 0.05 : 0);
const score = Math.min(1, base * recency + social);
common.sort((x, y) => weightOf(y) - weightOf(x) || x.localeCompare(y));
out.push({ ...strip(n), score, common, connections: common.length });
const author = String(n.author);
const flags = new Set();
if (!common.length && indirect > 0) flags.add('related');
if (common.some(t => contentTerms.has(t))) flags.add('content');
if (following.has(author) && supporters.has(author)) flags.add('mutual');
else if (following.has(author)) flags.add('following');
else if (supporters.has(author)) flags.add('supportsYou');
if (tribeMates.has(author)) flags.add('tribe');
const aff = affinity.get(n.author);
if (aff && aff.score >= ALIKE_MIN_AFFINITY) flags.add('alike');
if (common.some(t => cvTerms.has(t))) flags.add('cv');
if (common.some(t => pinnedTerms.has(t))) flags.add('pinned');
if (n.opinionsTotal >= RATED_MIN_OPINIONS) flags.add('rated');
if (myLocation && n.location && n.location === myLocation) flags.add('near');
const reasons = REASON_ORDER.filter(r => flags.has(r));
out.push({ ...strip(n), score, common, connections: common.length, reasons });
}
const anyMatches = out.length > 0;
const kindsAvail = {};
for (const n of use) {
if (String(n.author) !== String(viewerId)) kindsAvail[n.kind] = true;
}
if (f === 'RECENT') {
const cutoff = Date.now() - 7 * 24 * 60 * 60 * 1000;
const cutoff = Date.now() - 24 * 60 * 60 * 1000;
out = out.filter(s => s.ts >= cutoff);
} else if (f !== 'ALL' && f !== 'TOP') {
const kind = f.toLowerCase();
@ -200,14 +377,25 @@ module.exports = ({ cooler }) => {
const q = norm(opts.q);
if (q) out = out.filter(s => norm(s.title).includes(q) || s.common.some(t => t.includes(q)));
const present = new Set();
for (const s of out) for (const r of s.reasons) present.add(r);
const reasonsAvail = REASON_ORDER.filter(r => present.has(r));
const reason = String(opts.reason || '').trim();
if (reason && REASON_ORDER.includes(reason)) out = out.filter(s => s.reasons.includes(reason));
if (f === 'RECENT') out.sort((x, y) => y.ts - x.ts || y.score - x.score);
else out.sort((x, y) => y.score - x.score || y.ts - x.ts);
if (f === 'ALL' || f === 'TOP') out = diversify(out);
return {
matches: out.slice(0, MAX_PAIRS),
total: out.length,
hasProfile: myTermSet.size > 0,
myTerms: [...myTermSet]
myTerms: [...myTermSet],
kindsAvail,
anyMatches,
reasonsAvail,
reason: reasonsAvail.includes(reason) ? reason : ''
};
},

86
src/models/documents_model.js
View file

@ -2,7 +2,8 @@ const pull = require("../server/node_modules/pull-stream");
const { getConfig } = require("../configs/config-manager.js");
const categories = require("../backend/opinion_categories");
const { buildValidatedTombstoneSet } = require('./tombstone_validator');
const { dedupeBy, norm } = require('../backend/dedupe');
const { dedupeBy, norm, versionGroups } = require('../backend/dedupe');
const { readTyped, memoIndex } = require("./typed_log");
const contentFavorites = require("../backend/content_favorites");
const logLimit = getConfig().ssbLogStream?.limit || 1000;
@ -34,20 +35,13 @@ module.exports = ({ cooler }) => {
return ssb;
};
const getAllMessages = async (ssbClient) =>
new Promise((resolve, reject) => {
pull(
ssbClient.createLogStream({ limit: logLimit }),
pull.collect((err, msgs) => (err ? reject(err) : resolve(msgs)))
);
});
const DOCUMENT_TYPES = ["document", "documentOpinion", "tombstone"];
const getAllMessages = async (ssbClient) => readTyped(ssbClient, DOCUMENT_TYPES, { limit: logLimit });
const buildIndex = (messages) => {
const tomb = buildValidatedTombstoneSet(messages);
const nodes = new Map();
const parent = new Map();
const child = new Map();
const strictChild = new Map();
const opinionMsgs = [];
for (const m of messages) {
@ -63,50 +57,16 @@ module.exports = ({ cooler }) => {
const ts = v.timestamp || m.timestamp || 0;
let sizeBytes = 0;
try { sizeBytes = Buffer.byteLength(JSON.stringify(v), "utf8"); } catch (_) { sizeBytes = 0; }
nodes.set(k, { key: k, ts, c, sizeBytes, author: v.author });
if (c.replaces) {
parent.set(k, c.replaces);
child.set(c.replaces, k);
}
nodes.set(k, { key: k, ts, seq: v.sequence || 0, c, sizeBytes, author: v.author });
}
for (const [k, t] of Array.from(parent.entries())) {
const orig = nodes.get(t);
const node = nodes.get(k);
if (orig && node && orig.author === node.author) continue;
parent.delete(k);
if (child.get(t) === k) child.delete(t);
if (orig) nodes.delete(k);
}
for (const [k, node] of nodes) {
const t = node.c.replaces;
if (t) { const orig = nodes.get(t); if (orig && orig.author === node.author) strictChild.set(t, k); }
}
const rootOf = (id) => {
let cur = id, g = 0;
while (parent.has(cur) && nodes.has(parent.get(cur)) && g++ < 100000) cur = parent.get(cur);
return cur;
};
const contentTipOf = (root) => {
let cur = root, g = 0;
while (strictChild.has(cur) && g++ < 100000) cur = strictChild.get(cur);
const n = nodes.get(cur), rn = nodes.get(root);
return (n && rn && n.author === rn.author) ? cur : root;
};
const roots = new Set();
for (const id of nodes.keys()) roots.add(rootOf(id));
const vg = versionGroups([...nodes.values()].map(n => ({ key: n.key, author: n.author, seq: n.seq, ts: n.ts, replaces: n.c.replaces || null, claimedAuthor: typeof n.c.author === "string" ? n.c.author : null })));
const rootOf = vg.rootOf;
const contentTipOf = vg.tipOf;
const opinionsByRoot = new Map();
for (const op of opinionMsgs) { if (!nodes.has(op.target)) continue; const r = rootOf(op.target); if (!opinionsByRoot.has(r)) opinionsByRoot.set(r, []); opinionsByRoot.get(r).push(op); }
const forward = new Map();
for (const [newId, oldId] of parent.entries()) forward.set(oldId, newId);
const resolveGroup = (root) => {
const contentTip = contentTipOf(root);
const contentNode = nodes.get(contentTip) || nodes.get(root);
@ -123,9 +83,9 @@ module.exports = ({ cooler }) => {
};
const tipByRoot = new Map();
for (const r of roots) tipByRoot.set(r, contentTipOf(r));
for (const r of vg.roots) tipByRoot.set(r, contentTipOf(r));
return { tomb, nodes, parent, child, rootOf, tipByRoot, forward, resolveGroup };
return { tomb, nodes, rootOf, tipByRoot, resolveGroup };
};
const pickDoc = (node, rootId, agg) => {
@ -140,6 +100,7 @@ module.exports = ({ cooler }) => {
author: node.author,
title: c.title || "",
description: c.description || "",
license: c.license || "",
opinions: agg ? agg.opinions : (c.opinions || {}),
opinions_inhabitants: agg ? agg.voters : safeArr(c.opinions_inhabitants),
sizeBytes: node.sizeBytes || 0
@ -178,16 +139,12 @@ module.exports = ({ cooler }) => {
const messages = await getAllMessages(ssbClient);
const idx = buildIndex(messages);
let tip = id;
while (idx.forward.has(tip)) tip = idx.forward.get(tip);
if (idx.tomb.has(tip)) throw new Error("Document not found");
let root = tip;
while (idx.parent.has(root)) root = idx.parent.get(root);
const root = idx.rootOf(id);
if (idx.tomb.has(idx.tipByRoot.get(root) || id)) throw new Error("Document not found");
return root;
},
async createDocument(blobMarkdown, tagsRaw, title, description) {
async createDocument(blobMarkdown, tagsRaw, title, description, license) {
const ssbClient = await openSsb();
const blobId = parseBlobId(blobMarkdown);
if (!blobId) throw new Error("Missing document blob");
@ -203,7 +160,8 @@ module.exports = ({ cooler }) => {
title: title || "",
description: description || "",
opinions: {},
opinions_inhabitants: []
opinions_inhabitants: [],
...(license ? { license } : {})
};
return new Promise((resolve, reject) => {
@ -211,7 +169,7 @@ module.exports = ({ cooler }) => {
});
},
async updateDocumentById(id, blobMarkdown, tagsRaw, title, description) {
async updateDocumentById(id, blobMarkdown, tagsRaw, title, description, license) {
const ssbClient = await openSsb();
const tipId = await this.resolveCurrentId(id);
@ -238,7 +196,8 @@ module.exports = ({ cooler }) => {
tags,
title: title !== undefined ? (title || "") : oldMsg.content.title || "",
description: description !== undefined ? (description || "") : oldMsg.content.description || "",
updatedAt
updatedAt,
...(license !== undefined ? { license } : {})
};
const tombstone = { type: "tombstone", target: tipId, deletedAt: updatedAt, author: userId };
@ -327,12 +286,12 @@ module.exports = ({ cooler }) => {
async getDocumentById(id) {
const ssbClient = await openSsb();
const messages = await getAllMessages(ssbClient);
const idx = buildIndex(messages);
const idx = memoIndex('documents', messages, buildIndex);
const favorites = await favoritesSetForDocuments();
const rootId = idx.rootOf(id);
const agg = idx.resolveGroup(rootId);
if (!agg.contentNode || idx.tomb.has(agg.contentTip)) throw new Error("Document not found");
if (!idx.tipByRoot.has(rootId) || !agg.contentNode || idx.tomb.has(agg.contentTip)) throw new Error("Document not found");
const c = agg.contentNode.c || {};
return {
@ -345,6 +304,7 @@ module.exports = ({ cooler }) => {
author: agg.contentNode.author,
title: c.title || "",
description: c.description || "",
license: c.license || "",
opinions: agg.opinions,
opinions_inhabitants: agg.voters,
isFavorite: favorites.has(rootId || agg.contentTip)

297
src/models/emergencies_model.js Normal file
View file

@ -0,0 +1,297 @@
const { readTyped, memoIndex } = require('./typed_log');
const { buildValidatedTombstoneSet } = require('./tombstone_validator');
const { getConfig } = require('../configs/config-manager.js');
const logLimit = getConfig().ssbLogStream?.limit || 1000;
const EMERGENCY_TYPES = ['emergency', 'emergencyConfirm', 'emergencyUpdate', 'tombstone'];
const CATEGORIES = ['WEATHER', 'INFRASTRUCTURE', 'HEALTH', 'SECURITY', 'LOST', 'NEIGHBORHOOD'];
const SEVERITY_THRESHOLDS = { LOW: 2, MEDIUM: 5, HIGH: 10 };
const EXPIRES_OPTIONS = { '1d': 1, '3d': 3, '7d': 7, '30d': 30 };
const MEDIA_RE = /!?\[(image|video):[^\]]*\]\((&[^)]+)\)/;
const RECENT_MS = 24 * 60 * 60 * 1000;
const normU = (v) => String(v == null ? '' : v).trim().toUpperCase();
const safeText = (v) => String(v == null ? '' : v).trim();
const normalizeTags = (raw) => {
if (!raw) return [];
const arr = Array.isArray(raw) ? raw : String(raw).split(',');
return Array.from(new Set(arr.map(t => String(t || '').trim()).filter(Boolean)));
};
const normalizeCategory = (v) => (CATEGORIES.includes(normU(v)) ? normU(v) : 'NEIGHBORHOOD');
const severityOf = (confirmations) => {
const n = Number(confirmations) || 0;
if (n >= SEVERITY_THRESHOLDS.HIGH) return 'HIGH';
if (n >= SEVERITY_THRESHOLDS.MEDIUM) return 'MEDIUM';
if (n >= SEVERITY_THRESHOLDS.LOW) return 'LOW';
return 'UNVERIFIED';
};
const expiresAtFrom = (expiresIn, from = Date.now()) => {
const days = EXPIRES_OPTIONS[String(expiresIn || '').toLowerCase()];
return days ? new Date(from + days * 24 * 60 * 60 * 1000).toISOString() : null;
};
module.exports = ({ cooler }) => {
let ssb;
const openSsb = async () => { if (!ssb) ssb = await cooler.open(); return ssb; };
const readAll = async (ssbClient) => readTyped(ssbClient, EMERGENCY_TYPES, { limit: logLimit });
const buildIndex = (messages) => {
const tomb = buildValidatedTombstoneSet(messages);
const nodes = new Map();
const parentOf = new Map();
const confirms = [];
const updates = [];
for (const m of messages) {
const v = m.value || {};
const c = v.content;
if (!c || typeof c !== 'object') continue;
if (c.type === 'emergency') {
nodes.set(m.key, { key: m.key, author: v.author, ts: v.timestamp || m.timestamp || 0, c });
if (typeof c.replaces === 'string') parentOf.set(m.key, c.replaces);
} else if (c.type === 'emergencyConfirm' && typeof c.target === 'string') {
confirms.push({ key: m.key, target: c.target, author: v.author, ts: v.timestamp || 0 });
} else if (c.type === 'emergencyUpdate' && typeof c.target === 'string') {
updates.push({ key: m.key, target: c.target, author: v.author, ts: v.timestamp || 0, text: safeText(c.text), createdAt: c.createdAt || new Date(v.timestamp || 0).toISOString(), replaces: typeof c.replaces === 'string' ? c.replaces : null });
}
}
const rootOf = (key) => {
const seen = new Set();
let cur = key;
while (parentOf.has(cur) && !seen.has(cur)) { seen.add(cur); cur = parentOf.get(cur); }
return cur;
};
const updateParent = new Map(updates.filter(u => u.replaces).map(u => [u.key, u.replaces]));
const updateRootOf = (key) => {
const seen = new Set();
let cur = key;
while (updateParent.has(cur) && !seen.has(cur)) { seen.add(cur); cur = updateParent.get(cur); }
return cur;
};
const updateByKey = new Map(updates.map(u => [u.key, u]));
const updateVersions = new Map();
for (const u of updates) {
const uroot = updateRootOf(u.key);
if (tomb.has(uroot)) continue;
if (!updateVersions.has(uroot)) updateVersions.set(uroot, []);
updateVersions.get(uroot).push(u);
}
const updateItems = [];
for (const [uroot, versions] of updateVersions) {
versions.sort((a, b) => a.ts - b.ts);
const first = versions[0];
const own = versions.filter(v => v.author === first.author);
const tip = own[own.length - 1];
const confirmers = Array.from(new Set(confirms.filter(cf => updateByKey.has(cf.target) && updateRootOf(cf.target) === uroot && cf.author !== first.author).map(cf => cf.author)));
updateItems.push({
id: uroot, key: uroot, tipId: tip.key, target: first.target, author: first.author,
ts: first.ts, editedTs: tip.ts, text: tip.text, createdAt: first.createdAt, updatedAt: tip.createdAt, edited: tip.key !== uroot,
confirmations: confirmers, confirmationCount: confirmers.length,
lastActivityTs: Math.max(tip.ts, ...confirms.filter(cf => updateByKey.has(cf.target) && updateRootOf(cf.target) === uroot).map(cf => cf.ts))
});
}
const versionsByRoot = new Map();
for (const node of nodes.values()) {
const root = rootOf(node.key);
if (tomb.has(root)) continue;
if (!versionsByRoot.has(root)) versionsByRoot.set(root, []);
versionsByRoot.get(root).push(node);
}
const emergencies = new Map();
for (const [root, versions] of versionsByRoot) {
versions.sort((a, b) => a.ts - b.ts);
const first = versions[0];
const own = versions.filter(v => v.author === first.author);
const tip = own[own.length - 1];
const c = tip.c;
const thread = updateItems.filter(u => nodes.has(u.target) && rootOf(u.target) === root && u.author === first.author).sort((a, b) => a.ts - b.ts);
const confirmers = Array.from(new Set([
...confirms.filter(cf => nodes.has(cf.target) && rootOf(cf.target) === root && cf.author !== first.author).map(cf => cf.author),
...thread.flatMap(u => u.confirmations)
]));
const expiresAt = c.expiresAt || null;
const expired = !!(expiresAt && Date.parse(expiresAt) < Date.now());
const status = normU(c.status) === 'RESOLVED' ? 'RESOLVED' : (expired ? 'EXPIRED' : 'ACTIVE');
emergencies.set(root, {
id: root, rootId: root, tipId: tip.key,
title: safeText(c.title), text: safeText(c.text),
media: (() => { const m = safeText(c.text).match(MEDIA_RE); return m ? { kind: m[1], blobId: m[2] } : null; })(),
category: normalizeCategory(c.category), mapUrl: safeText(c.mapUrl), tags: normalizeTags(c.tags),
author: first.author, createdAt: first.c.createdAt || new Date(first.ts).toISOString(),
updatedAt: c.updatedAt || new Date(tip.ts).toISOString(), ts: tip.ts, lastActivityTs: Math.max(tip.ts, ...thread.map(u => u.lastActivityTs), ...confirms.filter(cf => nodes.has(cf.target) && rootOf(cf.target) === root).map(cf => cf.ts)),
expiresAt, expired, status, resolved: status === 'RESOLVED',
confirmations: confirmers, confirmationCount: confirmers.length, severity: severityOf(confirmers.length),
updates: thread
});
}
const updatesById = new Map(updateItems.map(u => [u.id, u]));
return { emergencies, rootOf, nodes, updatesById, updateRootOf };
};
const load = async () => {
const ssbClient = await openSsb();
const idx = memoIndex('emergencies', await readAll(ssbClient), buildIndex);
return { ssbClient, idx };
};
const find = (idx, id) => {
const key = String(id || '');
if (idx.emergencies.has(key)) return idx.emergencies.get(key);
if (idx.nodes.has(key)) return idx.emergencies.get(idx.rootOf(key)) || null;
return null;
};
const publish = (ssbClient, content) => new Promise((resolve, reject) => ssbClient.publish(content, (err, res) => err ? reject(err) : resolve(res)));
return {
CATEGORIES,
SEVERITY_THRESHOLDS,
EXPIRES_OPTIONS,
severityOf,
async createEmergency({ title, text, category, mapUrl, tags, expiresIn }) {
const ssbClient = await openSsb();
const cleanTitle = safeText(title);
if (!cleanTitle) throw new Error('Title required');
const now = new Date().toISOString();
const content = {
type: 'emergency',
title: cleanTitle,
text: safeText(text),
category: normalizeCategory(category),
mapUrl: safeText(mapUrl),
tags: normalizeTags(tags),
status: 'ACTIVE',
expiresAt: expiresAtFrom(expiresIn),
author: ssbClient.id,
createdAt: now,
updatedAt: now
};
return publish(ssbClient, content);
},
async updateEmergency(id, data = {}) {
const { ssbClient, idx } = await load();
const emergency = find(idx, id);
if (!emergency) throw new Error('Emergency not found');
if (String(emergency.author) !== String(ssbClient.id)) throw new Error('Only the author can update');
const editsContent = ['title', 'text', 'category', 'mapUrl', 'tags', 'expiresIn'].some(k => data[k] !== undefined);
if (editsContent && (emergency.confirmationCount || 0) > 0) throw new Error('Cannot edit an emergency after it has been confirmed');
const content = {
type: 'emergency',
title: data.title !== undefined ? (safeText(data.title) || emergency.title) : emergency.title,
text: data.text !== undefined ? safeText(data.text) : emergency.text,
category: data.category !== undefined ? normalizeCategory(data.category) : emergency.category,
mapUrl: data.mapUrl !== undefined ? safeText(data.mapUrl) : emergency.mapUrl,
tags: data.tags !== undefined ? normalizeTags(data.tags) : emergency.tags,
status: data.status !== undefined ? (normU(data.status) === 'RESOLVED' ? 'RESOLVED' : 'ACTIVE') : (emergency.resolved ? 'RESOLVED' : 'ACTIVE'),
expiresAt: data.expiresIn !== undefined && data.expiresIn !== 'keep' ? expiresAtFrom(data.expiresIn) : emergency.expiresAt,
author: ssbClient.id,
createdAt: emergency.createdAt,
updatedAt: new Date().toISOString(),
replaces: emergency.tipId
};
const res = await publish(ssbClient, content);
return { key: res.key, rootId: emergency.id };
},
async resolveEmergency(id) {
return this.updateEmergency(id, { status: 'RESOLVED' });
},
async confirmEmergency(id) {
const { ssbClient, idx } = await load();
const emergency = find(idx, id);
if (!emergency) throw new Error('Emergency not found');
if (String(emergency.author) === String(ssbClient.id)) throw new Error('Cannot confirm own emergency');
if (emergency.confirmations.includes(ssbClient.id)) throw new Error('Already confirmed');
return publish(ssbClient, { type: 'emergencyConfirm', target: emergency.id, createdAt: new Date().toISOString() });
},
async confirmUpdate(updateId) {
const { ssbClient, idx } = await load();
const update = idx.updatesById.get(idx.updateRootOf(String(updateId || '')));
if (!update) throw new Error('Update not found');
if (String(update.author) === String(ssbClient.id)) throw new Error('Cannot confirm own update');
if (update.confirmations.includes(ssbClient.id)) throw new Error('Already confirmed');
return publish(ssbClient, { type: 'emergencyConfirm', target: update.id, createdAt: new Date().toISOString() });
},
async editUpdate(updateId, text) {
const { ssbClient, idx } = await load();
const update = idx.updatesById.get(idx.updateRootOf(String(updateId || '')));
if (!update) throw new Error('Update not found');
if (String(update.author) !== String(ssbClient.id)) throw new Error('Only the author can edit an update');
if (update.confirmationCount > 0) throw new Error('Cannot edit an update after it has been confirmed');
const clean = safeText(text);
if (!clean) throw new Error('Text required');
const res = await publish(ssbClient, { type: 'emergencyUpdate', target: update.target, text: clean, createdAt: new Date().toISOString(), replaces: update.tipId });
return { key: res.key, rootId: update.id, emergencyId: update.target };
},
async deleteUpdate(updateId) {
const { ssbClient, idx } = await load();
const update = idx.updatesById.get(idx.updateRootOf(String(updateId || '')));
if (!update) throw new Error('Update not found');
if (String(update.author) !== String(ssbClient.id)) throw new Error('Only the author can delete an update');
await publish(ssbClient, { type: 'tombstone', target: update.id, deletedAt: new Date().toISOString(), author: ssbClient.id });
return { key: update.id, emergencyId: update.target };
},
async addUpdate(id, text) {
const { ssbClient, idx } = await load();
const emergency = find(idx, id);
if (!emergency) throw new Error('Emergency not found');
if (String(emergency.author) !== String(ssbClient.id)) throw new Error('Only the author can post updates');
const clean = safeText(text);
if (!clean) throw new Error('Text required');
return publish(ssbClient, { type: 'emergencyUpdate', target: emergency.id, text: clean, createdAt: new Date().toISOString() });
},
async deleteEmergency(id) {
const { ssbClient, idx } = await load();
const emergency = find(idx, id);
if (!emergency) throw new Error('Emergency not found');
if (String(emergency.author) !== String(ssbClient.id)) throw new Error('Only the author can delete');
return publish(ssbClient, { type: 'tombstone', target: emergency.id, deletedAt: new Date().toISOString(), author: ssbClient.id });
},
async getEmergencyById(id) {
const { idx } = await load();
return find(idx, id);
},
async listAll({ filter = 'all', q = '', viewerId } = {}) {
const { ssbClient, idx } = await load();
const me = viewerId || ssbClient.id;
let list = Array.from(idx.emergencies.values());
const f = normU(filter || 'all');
if (f === 'MINE') list = list.filter(a => String(a.author) === String(me));
else if (f === 'RECENT') list = list.filter(a => a.lastActivityTs >= Date.now() - RECENT_MS);
else if (f === 'ACTIVE') list = list.filter(a => a.status === 'ACTIVE');
else if (f === 'RESOLVED') list = list.filter(a => a.status === 'RESOLVED');
else if (f === 'EXPIRED') list = list.filter(a => a.status === 'EXPIRED');
else if (['UNVERIFIED', 'LOW', 'MEDIUM', 'HIGH'].includes(f)) list = list.filter(a => a.severity === f);
else if (CATEGORIES.includes(f)) list = list.filter(a => a.category === f);
const needle = safeText(q).toLowerCase();
if (needle) list = list.filter(a => a.title.toLowerCase().includes(needle) || a.text.toLowerCase().includes(needle) || a.tags.some(t => t.toLowerCase().includes(needle)));
return list.sort((a, b) => b.lastActivityTs - a.lastActivityTs);
},
async featured() {
const { idx } = await load();
const rank = { HIGH: 3, MEDIUM: 2, LOW: 1, UNVERIFIED: 0 };
const candidates = Array.from(idx.emergencies.values()).filter(a => a.status === 'ACTIVE' && a.severity === 'HIGH');
candidates.sort((a, b) => (rank[b.severity] - rank[a.severity]) || (b.confirmationCount - a.confirmationCount) || (b.lastActivityTs - a.lastActivityTs));
const top = candidates[0];
return top ? { id: top.id, title: top.title, severity: top.severity, category: top.category, confirmationCount: top.confirmationCount, href: `/emergencies/${encodeURIComponent(top.id)}` } : null;
},
async resolveRootId(id) {
const { idx } = await load();
const emergency = find(idx, id);
if (!emergency) throw new Error('Not found');
return emergency.id;
}
};
};

301
src/models/events_model.js
View file

@ -5,9 +5,17 @@ const { truthy, hasAnyInterval, nextOccurrence, upcomingOccurrences } = require(
const crypto = require('crypto');
const { buildValidatedTombstoneSet } = require('./tombstone_validator');
const { getConfig } = require('../configs/config-manager.js');
const { dedupeBy, norm } = require('../backend/dedupe');
const { dedupeBy, norm, versionGroups } = require('../backend/dedupe');
const { readTyped, memoIndex } = require('./typed_log');
const logLimit = getConfig().ssbLogStream?.limit || 1000;
const EVENT_TYPES = [
'event', 'eventOpinion', 'eventAttend',
'event-invite', 'event-invite-tombstone',
'event-open-invite', 'event-open-invite-tombstone',
'tribe-keys', 'tombstone'
];
module.exports = ({ cooler, tribeCrypto, eventCrypto, tribesModel }) => {
let ssb;
const openSsb = async () => { if (!ssb) ssb = await cooler.open(); return ssb; };
@ -16,10 +24,53 @@ module.exports = ({ cooler, tribeCrypto, eventCrypto, tribesModel }) => {
const ownCrypto = eventCrypto || tribeCrypto;
const lookupKey = (rid) => (ownCrypto && ownCrypto.getKey(rid)) || (tribeCrypto && tribeCrypto.getKey(rid)) || null;
const lookupGen = (rid) => ((ownCrypto && ownCrypto.getGen(rid)) || (tribeCrypto && tribeCrypto.getGen(rid)) || 0);
const readAll = async (ssbClient) =>
new Promise((resolve, reject) =>
pull(ssbClient.createLogStream({ limit: logLimit }), pull.collect((err, msgs) => err ? reject(err) : resolve(msgs)))
);
readTyped(ssbClient, EVENT_TYPES, { limit: logLimit });
const rotateEventKey = async (rid, attendees) => {
if (!ownCrypto || !tribeCrypto || !rid || !lookupKey(rid)) return;
const ssbClient = await openSsb();
const ssbKeys = require("../server/node_modules/ssb-keys");
const newKey = ownCrypto.generateTribeKey();
const newGen = ownCrypto.addNewKey(rid, newKey);
const memberKeys = {};
for (const m of new Set([ssbClient.id, ...(Array.isArray(attendees) ? attendees : [])])) {
try { memberKeys[m] = tribeCrypto.boxKeyForMember(newKey, m, ssbKeys); } catch (_) {}
}
await new Promise((resolve) => {
ssbClient.publish({ type: "tribe-keys", tribeId: rid, generation: newGen, memberKeys }, () => resolve());
});
};
const ensureAttendeeKeys = async (ssbClient, messages, events) => {
if (!tribeCrypto || !ownCrypto) return;
const leavesByRoot = new Map();
for (const m of messages) {
const c = m.value && m.value.content;
if (!c || c.type !== 'eventAttend' || c.on !== false || typeof c.target !== 'string') continue;
if (!leavesByRoot.has(c.target)) leavesByRoot.set(c.target, []);
leavesByRoot.get(c.target).push({ member: m.value.author, ts: Number(m.timestamp || m.value.timestamp || 0) });
}
const ssbKeys = require("../server/node_modules/ssb-keys");
for (const ev of events) {
if (!ev || !ev.rid || ev.organizer !== ssbClient.id || !lookupKey(ev.rid)) continue;
const members = Array.isArray(ev.attendees) ? ev.attendees : [];
const plan = ownCrypto.keyPlan({ rootId: ev.rid, ownerId: ev.organizer, gen: lookupGen(ev.rid), members, messages, leaves: leavesByRoot.get(ev.rid) || [] });
if (plan.rotate) { await rotateEventKey(ev.rid, members); continue; }
if (!plan.missing.length) continue;
const key = lookupKey(ev.rid);
const memberKeys = {};
for (const m of plan.missing) {
try { memberKeys[m] = tribeCrypto.boxKeyForMember(key, m, ssbKeys); } catch (_) {}
}
if (!Object.keys(memberKeys).length) continue;
await new Promise((resolve) => {
ssbClient.publish({ type: "tribe-keys", tribeId: ev.rid, generation: lookupGen(ev.rid) || 1, memberKeys }, () => resolve());
});
}
};
const ingestOwnTribeKeys = async () => {
if (!ownCrypto) return;
@ -28,9 +79,11 @@ module.exports = ({ cooler, tribeCrypto, eventCrypto, tribesModel }) => {
const ssbKeys = require("../server/node_modules/ssb-keys");
const cfg = require("../server/ssb_config");
const msgs = await readAll(ssbClient);
const idx = buildEventIndex(msgs);
for (const m of msgs) {
const c = m.value && m.value.content;
if (!c || c.type !== "tribe-keys") continue;
if (!c.tribeId || (m.value.author !== idx.eventAuthor.get(c.tribeId) && m.value.author !== ssbClient.id)) continue;
const memberKeys = c.memberKeys;
if (!memberKeys || typeof memberKeys !== "object") continue;
const boxed = memberKeys[ssbClient.id];
@ -98,45 +151,18 @@ module.exports = ({ cooler, tribeCrypto, eventCrypto, tribesModel }) => {
const buildEventIndex = (messages) => {
const eventAuthor = new Map();
const eventReplaces = new Map();
const entries = [];
for (const m of messages) {
const c = m.value && m.value.content;
if (!c || c.type !== 'event') continue;
eventAuthor.set(m.key, m.value.author);
if (c.replaces) eventReplaces.set(m.key, c.replaces);
entries.push({ key: m.key, author: m.value.author, seq: m.value.sequence || 0, ts: m.value.timestamp || m.timestamp || 0, replaces: typeof c.replaces === 'string' ? c.replaces : null, claimedAuthor: typeof c.organizer === 'string' ? c.organizer : null });
}
for (const [key, target] of Array.from(eventReplaces.entries())) {
if (!eventAuthor.has(target)) { eventReplaces.delete(key); continue; }
if (eventAuthor.get(target) !== eventAuthor.get(key)) { eventReplaces.delete(key); eventAuthor.delete(key); }
}
const strictNext = new Map();
for (const [key, target] of eventReplaces) {
if (eventAuthor.get(target) === eventAuthor.get(key)) strictNext.set(target, key);
}
const rootOf = (id) => {
let cur = id;
const seen = new Set();
while (eventReplaces.has(cur) && eventAuthor.has(eventReplaces.get(cur))) {
if (seen.has(cur)) break;
seen.add(cur);
cur = eventReplaces.get(cur);
}
return cur;
};
const contentTipOf = (root) => {
let cur = root;
const seen = new Set();
while (strictNext.has(cur)) {
if (seen.has(cur)) break;
seen.add(cur);
cur = strictNext.get(cur);
}
return cur;
};
return { eventAuthor, eventReplaces, strictNext, rootOf, contentTipOf };
const vg = versionGroups(entries);
return { eventAuthor, vg, rootOf: vg.rootOf, contentTipOf: vg.tipOf };
};
const collectCollab = (messages) => {
const collectCollab = (messages, rootOf = (id) => id) => {
const opinions = new Map();
const attend = new Map();
for (const m of messages) {
@ -144,13 +170,14 @@ module.exports = ({ cooler, tribeCrypto, eventCrypto, tribesModel }) => {
if (!c || typeof c.target !== 'string') continue;
const author = m.value.author;
const ts = (m.value && m.value.timestamp) || 0;
const target = rootOf(c.target);
if (c.type === 'eventOpinion' && typeof c.category === 'string') {
if (!opinions.has(c.target)) opinions.set(c.target, new Map());
const byAuthor = opinions.get(c.target);
if (!opinions.has(target)) opinions.set(target, new Map());
const byAuthor = opinions.get(target);
if (!byAuthor.has(author)) byAuthor.set(author, c.category);
} else if (c.type === 'eventAttend') {
if (!attend.has(c.target)) attend.set(c.target, new Map());
const byAuthor = attend.get(c.target);
if (!attend.has(target)) attend.set(target, new Map());
const byAuthor = attend.get(target);
const prev = byAuthor.get(author);
if (!prev || ts >= prev.ts) byAuthor.set(author, { on: c.on !== false, ts });
}
@ -285,23 +312,28 @@ module.exports = ({ cooler, tribeCrypto, eventCrypto, tribesModel }) => {
const ssbClient = await openSsb();
const rid = await this.resolveRootId(eventId).catch(() => eventId);
const messages = await readAll(ssbClient);
const authorOf = new Map();
for (const m of messages) if (m && m.key && m.value) authorOf.set(m.key, m.value.author);
const eventAuthor = authorOf.get(rid);
if (!eventAuthor) return null;
const markerTomb = new Set();
const invTomb = new Set();
for (const m of messages) {
const c = m.value && m.value.content;
if (!c) continue;
if (c.type === 'event-open-invite-tombstone' && typeof c.target === 'string') markerTomb.add(c.target);
if (c.type === 'event-invite-tombstone' && typeof c.target === 'string') invTomb.add(c.target);
if (!c || typeof c.target !== 'string' || authorOf.get(c.target) !== m.value.author) continue;
if (c.type === 'event-open-invite-tombstone') markerTomb.add(c.target);
if (c.type === 'event-invite-tombstone') invTomb.add(c.target);
}
let best = null;
for (const m of messages) {
const c = m.value && m.value.content;
if (!c || c.type !== 'event-open-invite' || c.v !== 1) continue;
if (c.target !== rid || typeof c.code !== 'string') continue;
if (m.value.author !== eventAuthor) continue;
if (markerTomb.has(m.key)) continue;
if (c.inviteKey && invTomb.has(c.inviteKey)) continue;
const ts = (m.value && m.value.timestamp) || 0;
if (!best || ts > best.ts) best = { code: c.code, by: c.by || m.value.author, markerKey: m.key, inviteKey: c.inviteKey || null, ts };
if (!best || ts > best.ts) best = { code: c.code, by: m.value.author, markerKey: m.key, inviteKey: c.inviteKey || null, ts };
}
return best ? { code: best.code, by: best.by, markerKey: best.markerKey, inviteKey: best.inviteKey } : null;
},
@ -346,6 +378,16 @@ module.exports = ({ cooler, tribeCrypto, eventCrypto, tribesModel }) => {
if (rec.by !== userId && organizer !== userId) throw new Error("Not allowed to remove this invitation");
await new Promise((resolve, reject) => ssbClient.publish({ type: 'event-open-invite-tombstone', target: rec.markerKey, ts: new Date().toISOString() }, (err) => err ? reject(err) : resolve()));
if (rec.inviteKey) await new Promise((resolve, reject) => ssbClient.publish({ type: 'event-invite-tombstone', target: rec.inviteKey, ts: new Date().toISOString() }, (err) => err ? reject(err) : resolve()));
if (organizer === userId) {
const rid = await this.resolveRootId(eventId);
const messages = await readAll(ssbClient);
const idx = buildEventIndex(messages);
const tip = idx.contentTipOf(rid);
const raw = messages.find(x => x.key === tip);
const rc = raw && raw.value && raw.value.content;
const dec = rc && rc.encryptedPayload ? decryptEventContent(rc, rid) : rc;
if (dec && !dec._undecryptable) await rotateEventKey(rid, aggregateCollab(dec, rid, collectCollab(messages, idx.rootOf)).attendees);
}
},
async joinByInvite(code) {
@ -390,14 +432,13 @@ module.exports = ({ cooler, tribeCrypto, eventCrypto, tribesModel }) => {
const status = deriveStatus(c);
if (status === 'CLOSED') throw new Error("Cannot attend a closed event");
const collab = collectCollab(messages);
const collab = collectCollab(messages, idx.rootOf);
const agg = aggregateCollab(c, rid, collab);
const isAttending = agg.attendees.includes(userId);
const isLeaving = isAttending;
const on = !isAttending;
const isPrivate = normalizePrivacy(c.isPublic) === 'private';
const isOrganizer = c.organizer === userId;
const result = await new Promise((resolve, reject) => {
ssbClient.publish({ type: 'eventAttend', target: rid, on, createdAt: new Date().toISOString() }, (err2, res2) => err2 ? reject(err2) : resolve(res2));
@ -419,36 +460,20 @@ module.exports = ({ cooler, tribeCrypto, eventCrypto, tribesModel }) => {
} catch (_) {}
}
if (isPrivate && isLeaving && !isOrganizer && ownCrypto && tribeCrypto) {
try {
const remaining = agg.attendees.filter(a => a !== userId);
const newKey = ownCrypto.generateTribeKey();
const newGen = ownCrypto.addNewKey(rid, newKey);
const ssbKeys = require("../server/node_modules/ssb-keys");
const memberKeys = {};
for (const m of remaining) {
try { memberKeys[m] = tribeCrypto.boxKeyForMember(newKey, m, ssbKeys); } catch (_) {}
}
if (Object.keys(memberKeys).length) {
await new Promise((resolve) => {
ssbClient.publish({ type: "tribe-keys", tribeId: rid, generation: newGen, memberKeys }, () => resolve());
});
}
} catch (_) {}
}
return result;
},
async deleteEventById(eventId) {
const ssbClient = await openSsb();
const userId = await me();
const ev = await new Promise((res, rej) => ssbClient.get(eventId, (err, ev) => err || !ev || !ev.content ? rej(new Error("Error retrieving event")) : res(ev)));
const rid = await this.resolveRootId(eventId);
const idx = buildEventIndex(await readAll(ssbClient));
const rid = idx.rootOf(eventId);
const tipId = idx.contentTipOf(rid);
const ev = await new Promise((res, rej) => ssbClient.get(tipId, (err, ev) => err || !ev || !ev.content ? rej(new Error("Error retrieving event")) : res(ev)));
const c = ev.content && ev.content.encryptedPayload ? decryptEventContent(ev.content, rid) : ev.content;
if (c && c._undecryptable) throw new Error("Event is encrypted and cannot be decrypted");
if (c.organizer !== userId) throw new Error("Only the organizer can delete this event");
const tombstone = { type: 'tombstone', target: eventId, deletedAt: new Date().toISOString(), author: userId };
if (c.organizer !== userId || ev.author !== userId) throw new Error("Only the organizer can delete this event");
const tombstone = { type: 'tombstone', target: tipId, deletedAt: new Date().toISOString(), author: userId };
return new Promise((resolve, reject) => {
ssbClient.publish(tombstone, (err, res) => err ? reject(err) : resolve(res));
});
@ -457,15 +482,19 @@ module.exports = ({ cooler, tribeCrypto, eventCrypto, tribesModel }) => {
async getEventById(eventId) {
const ssbClient = await openSsb();
const messages = await readAll(ssbClient);
const idx = buildEventIndex(messages);
const idx = memoIndex('events', messages, buildEventIndex);
const rid = idx.rootOf(eventId);
if (idx.vg.isForeignCopy(rid)) throw new Error("Error retrieving event");
const contentTip = idx.contentTipOf(rid);
const tombstoned = buildValidatedTombstoneSet(messages);
if (tombstoned.has(rid) || tombstoned.has(contentTip)) throw new Error("Error retrieving event");
const msg = await new Promise((res, rej) => ssbClient.get(contentTip, (err, msg) => err || !msg || !msg.content ? rej(new Error("Error retrieving event")) : res(msg)));
const c = msg.content && msg.content.encryptedPayload ? decryptEventContent(msg.content, rid) : msg.content;
if (c && c._undecryptable) throw new Error("Event is encrypted and cannot be decrypted with available keys");
if (c && c.organizer && c.organizer !== msg.author) throw new Error("Error retrieving event");
const status = deriveStatus(c);
const agg = aggregateCollab(c, rid, collectCollab(messages));
const agg = aggregateCollab(c, rid, collectCollab(messages, idx.rootOf));
return {
id: contentTip,
@ -502,11 +531,13 @@ module.exports = ({ cooler, tribeCrypto, eventCrypto, tribesModel }) => {
async updateEventById(eventId, updatedData) {
const ssbClient = await openSsb();
const userId = await me();
const ev = await new Promise((res, rej) => ssbClient.get(eventId, (err, ev) => err || !ev || !ev.content ? rej(new Error("Error retrieving event")) : res(ev)));
const rid = await this.resolveRootId(eventId);
const idx = buildEventIndex(await readAll(ssbClient));
const rid = idx.rootOf(eventId);
const tipId = idx.contentTipOf(rid);
const ev = await new Promise((res, rej) => ssbClient.get(tipId, (err, ev) => err || !ev || !ev.content ? rej(new Error("Error retrieving event")) : res(ev)));
const c = ev.content && ev.content.encryptedPayload ? decryptEventContent(ev.content, rid) : ev.content;
if (c && c._undecryptable) throw new Error("Event is encrypted and cannot be decrypted");
if (c.organizer !== userId) throw new Error("Only the organizer can update this event");
if (c.organizer !== userId || ev.author !== userId) throw new Error("Only the organizer can update this event");
const status = deriveStatus(c);
if (status === 'CLOSED') throw new Error("Cannot edit a closed event");
@ -541,7 +572,7 @@ module.exports = ({ cooler, tribeCrypto, eventCrypto, tribesModel }) => {
intervalYearly: updatedData.intervalYearly !== undefined ? truthy(updatedData.intervalYearly) : truthy(c.intervalYearly),
recurrenceUntil: updatedData.recurrenceUntil !== undefined ? String(updatedData.recurrenceUntil || '').trim() : (c.recurrenceUntil || ''),
updatedAt: new Date().toISOString(),
replaces: eventId
replaces: tipId
};
const wasPrivate = normalizePrivacy(c.isPublic) === 'private';
@ -591,7 +622,7 @@ module.exports = ({ cooler, tribeCrypto, eventCrypto, tribesModel }) => {
const ev = await new Promise((res, rej) => ssbClient.get(contentTip, (err, m) => (err || !m || !m.content) ? rej(new Error('Event not found')) : res(m)));
const c = ev.content && ev.content.encryptedPayload ? decryptEventContent(ev.content, rid) : ev.content;
if (c && c._undecryptable) throw new Error('Event is encrypted and cannot be decrypted');
const agg = aggregateCollab(c, rid, collectCollab(messages));
const agg = aggregateCollab(c, rid, collectCollab(messages, idx.rootOf));
if (agg.opinions_inhabitants.includes(userId)) throw new Error('Already opined');
return new Promise((res, rej) => ssbClient.publish({ type: 'eventOpinion', target: rid, category, createdAt: new Date().toISOString() }, (err, result) => err ? rej(err) : res(result)));
},
@ -599,79 +630,67 @@ module.exports = ({ cooler, tribeCrypto, eventCrypto, tribesModel }) => {
async listAll(author = null, filter = 'all') {
const ssbClient = await openSsb();
const userId = await me();
return new Promise((resolve, reject) => {
pull(
ssbClient.createLogStream({ limit: logLimit }),
pull.collect((err, results) => {
if (err) return reject(new Error("Error listing events: " + err.message));
const tombstoned = buildValidatedTombstoneSet(results);
const idx = buildEventIndex(results);
const collab = collectCollab(results);
const results = await readAll(ssbClient);
const tombstoned = buildValidatedTombstoneSet(results);
const idx = buildEventIndex(results);
const collab = collectCollab(results, idx.rootOf);
const roots = new Set();
for (const r of results) {
const rawC = r.value && r.value.content;
if (!rawC || rawC.type !== 'event') continue;
if (!idx.eventAuthor.has(r.key)) continue;
roots.add(idx.rootOf(r.key));
}
const byKey = new Map(results.map(x => [x.key, x]));
const byRoot = new Map();
for (const rid of idx.vg.roots) {
if (tombstoned.has(rid)) continue;
const contentTip = idx.contentTipOf(rid);
if (tombstoned.has(contentTip)) continue;
const r = byKey.get(contentTip);
if (!r) continue;
const rawC = r.value && r.value.content;
if (!rawC) continue;
const c = rawC.encryptedPayload ? decryptEventContent(rawC, rid) : rawC;
if (!c || c._undecryptable) continue;
if (author && c.organizer !== author) continue;
const byRoot = new Map();
for (const rid of roots) {
if (tombstoned.has(rid)) continue;
const contentTip = idx.contentTipOf(rid);
if (tombstoned.has(contentTip)) continue;
const r = results.find(x => x.key === contentTip);
if (!r) continue;
const rawC = r.value && r.value.content;
if (!rawC) continue;
const c = rawC.encryptedPayload ? decryptEventContent(rawC, rid) : rawC;
if (!c || c._undecryptable) continue;
if (author && c.organizer !== author) continue;
const status = deriveStatus(c);
const agg = aggregateCollab(c, rid, collab);
const status = deriveStatus(c);
const agg = aggregateCollab(c, rid, collab);
byRoot.set(rid, {
id: contentTip,
title: c.title || '',
description: c.description || '',
date: c.date || '',
location: c.location || '',
price: c.price || 0,
url: c.url || '',
attendees: agg.attendees,
tags: Array.isArray(c.tags) ? c.tags.filter(Boolean) : [],
createdAt: c.createdAt || new Date().toISOString(),
organizer: c.organizer || '',
status,
isPublic: normalizePrivacy(c.isPublic),
mapUrl: c.mapUrl || "",
images: normalizeImages(c.images),
video: normalizeVideo(c.video),
intervalWeekly: truthy(c.intervalWeekly),
intervalMonthly: truthy(c.intervalMonthly),
intervalYearly: truthy(c.intervalYearly),
recurrenceUntil: c.recurrenceUntil || '',
recurring: hasAnyInterval(truthy(c.intervalWeekly), truthy(c.intervalMonthly), truthy(c.intervalYearly)) && !!c.recurrenceUntil,
nextDate: effectiveDate(c),
encrypted: normalizePrivacy(c.isPublic) === 'private',
opinions: agg.opinions,
opinions_inhabitants: agg.opinions_inhabitants
});
}
byRoot.set(rid, {
id: contentTip,
title: c.title || '',
description: c.description || '',
date: c.date || '',
location: c.location || '',
price: c.price || 0,
url: c.url || '',
attendees: agg.attendees,
tags: Array.isArray(c.tags) ? c.tags.filter(Boolean) : [],
createdAt: c.createdAt || new Date().toISOString(),
organizer: c.organizer || '',
status,
isPublic: normalizePrivacy(c.isPublic),
mapUrl: c.mapUrl || "",
images: normalizeImages(c.images),
video: normalizeVideo(c.video),
intervalWeekly: truthy(c.intervalWeekly),
intervalMonthly: truthy(c.intervalMonthly),
intervalYearly: truthy(c.intervalYearly),
recurrenceUntil: c.recurrenceUntil || '',
recurring: hasAnyInterval(truthy(c.intervalWeekly), truthy(c.intervalMonthly), truthy(c.intervalYearly)) && !!c.recurrenceUntil,
nextDate: effectiveDate(c),
encrypted: normalizePrivacy(c.isPublic) === 'private',
opinions: agg.opinions,
opinions_inhabitants: agg.opinions_inhabitants
});
}
try { await ensureAttendeeKeys(ssbClient, results, [...byRoot].filter(([rid, e]) => e.encrypted && idx.eventAuthor.get(rid) === userId).map(([rid, e]) => ({ rid, organizer: userId, attendees: e.attendees }))); } catch (_) {}
let out = Array.from(byRoot.values());
out = dedupeBy(out, e => e.title ? [norm(e.organizer), norm(e.title), norm(e.date)].join('|') : null);
let out = Array.from(byRoot.values());
out = dedupeBy(out, e => e.title ? [norm(e.organizer), norm(e.title), norm(e.date)].join('|') : null);
if (filter === 'mine') out = out.filter(e => e.organizer === userId);
if (filter === 'open') out = out.filter(e => String(e.status).toUpperCase() === 'OPEN');
if (filter === 'closed') out = out.filter(e => String(e.status).toUpperCase() === 'CLOSED');
if (filter === 'mine') out = out.filter(e => e.organizer === userId);
if (filter === 'open') out = out.filter(e => String(e.status).toUpperCase() === 'OPEN');
if (filter === 'closed') out = out.filter(e => String(e.status).toUpperCase() === 'CLOSED');
resolve(out);
})
);
});
return out;
}
};
};

11
src/models/exportmode_model.js
View file

@ -1,15 +1,13 @@
const os = require('os');
const fs = require('fs');
const path = require('path');
const archiver = require('../server/node_modules/archiver');
module.exports = {
exportSSB: async (outputPath) => {
try {
// se exporta la cuenta activa, no siempre la de por defecto
const ssbPath = require('../server/ssb_config').path;
const output = fs.createWriteStream(outputPath);
const archive = archiver('zip', {
const ssbPath = require('../configs/state-manager').ssbDir();
const output = fs.createWriteStream(outputPath, { mode: 0o600 });
const archive = require('../server/node_modules/archiver')('zip', {
zlib: { level: 9 }
});
archive.pipe(output);
@ -22,7 +20,8 @@ module.exports = {
const filePath = path.join(dirPath, file);
const stat = fs.statSync(filePath);
if (file === 'secret') {
const rel = path.relative(ssbPath, filePath);
if (file === 'secret' || file.startsWith('secret.') || file === 'oasis-config.json' || rel === path.join('oasis', 'keys') || rel.startsWith(path.join('oasis', 'keys') + path.sep)) {
return;
}

44
src/models/favorites_model.js
View file

@ -15,7 +15,7 @@ const toTs = (d) => {
return Number.isFinite(t) ? t : 0;
};
module.exports = ({ audiosModel, bookmarksModel, documentsModel, imagesModel, videosModel, mapsModel, padsModel, chatsModel, calendarsModel, torrentsModel, marketModel, shopsModel, eventsModel, tasksModel, reportsModel, votesModel, jobsModel, housingModel, projectsModel, transfersModel, forumModel, blogsModel, pollsModel }) => {
module.exports = ({ audiosModel, bookmarksModel, documentsModel, imagesModel, videosModel, mapsModel, padsModel, roomsModel, chatsModel, calendarsModel, torrentsModel, filesModel, marketModel, shopsModel, eventsModel, tasksModel, reportsModel, votesModel, jobsModel, housingModel, projectsModel, transfersModel, forumModel, blogsModel, pollsModel, schoolModel, wikiModel, emergenciesModel, mailingModel, logisticsModel, podcastsModel, campaignsModel }) => {
const kindConfig = {
audios: {
base: "/audios/",
@ -45,6 +45,34 @@ module.exports = ({ audiosModel, bookmarksModel, documentsModel, imagesModel, vi
base: "/pads/",
getById: getFn(padsModel, ["getPadById", "getById"])
},
rooms: {
base: "/rooms/",
getById: getFn(roomsModel, ["getRoomById"])
},
wiki: {
base: "/wiki/",
getById: getFn(wikiModel, ["getPage"])
},
emergencies: {
base: "/emergencies/",
getById: getFn(emergenciesModel, ["getEmergencyById"])
},
mailing: {
base: "/mailing/",
getById: getFn(mailingModel, ["getListById"])
},
logistics: {
base: "/logistics/",
getById: getFn(logisticsModel, ["getRouteById"])
},
podcasts: {
base: "/podcasts/",
getById: getFn(podcastsModel, ["getAnyById"])
},
campaigns: {
base: "/campaigns/",
getById: getFn(campaignsModel, ["getCampaignById"])
},
chats: {
base: "/chats/",
getById: getFn(chatsModel, ["getChatById", "getById"])
@ -57,6 +85,10 @@ module.exports = ({ audiosModel, bookmarksModel, documentsModel, imagesModel, vi
base: "/torrents/",
getById: getFn(torrentsModel, ["getTorrentById", "getById"])
},
files: {
base: "/files/",
getById: getFn(filesModel, ["getFileById", "getById"])
},
market: {
base: "/market/",
getById: getFn(marketModel, ["getItemById", "getById"])
@ -77,6 +109,10 @@ module.exports = ({ audiosModel, bookmarksModel, documentsModel, imagesModel, vi
base: "/reports/",
getById: getFn(reportsModel, ["getReportById", "getById"])
},
school: {
base: "/school/course/",
getById: getFn(schoolModel, ["getCourseById", "getById"])
},
votes: {
base: "/votes/",
getById: getFn(votesModel, ["getVoteById", "getById"])
@ -115,7 +151,7 @@ module.exports = ({ audiosModel, bookmarksModel, documentsModel, imagesModel, vi
}
};
const kindOrder = ["audios", "blogs", "bookmarks", "calendars", "chats", "documents", "events", "forum", "housing", "images", "jobs", "maps", "market", "pads", "polls", "projects", "reports", "shopProducts", "shops", "tasks", "torrents", "transfers", "videos", "votes"];
const kindOrder = ["audios", "blogs", "bookmarks", "calendars", "campaigns", "chats", "documents", "events", "files", "forum", "housing", "images", "jobs", "logistics", "mailing", "maps", "market", "pads", "podcasts", "polls", "projects", "reports", "rooms", "school", "shopProducts", "shops", "emergencies", "tasks", "torrents", "transfers", "videos", "votes", "wiki"];
const hydrateKind = async (kind, ids) => {
const cfg = kindConfig[kind];
@ -172,9 +208,11 @@ module.exports = ({ audiosModel, bookmarksModel, documentsModel, imagesModel, vi
for (const k of kindOrder) counts[k] = (byKind[k] || []).length;
const recentFlat = flat
.slice()
.filter((x) => (toTs(x.updatedAt) || toTs(x.createdAt) || 0) >= Date.now() - 86400000)
.sort((a, b) => (toTs(b.updatedAt) || toTs(b.createdAt)) - (toTs(a.updatedAt) || toTs(a.createdAt)));
counts.recent = recentFlat.length;
return { byKind, flat, recentFlat, counts };
};

764
src/models/fediverse_model.js
View file

@ -1,8 +1,9 @@
const fs = require('fs');
const path = require('path');
const { Readable } = require('stream');
const { getConfig, saveConfig } = require('../configs/config-manager.js');
const ACCOUNTS_PATH = path.join(__dirname, '..', 'configs', 'fediverse-accounts.json');
const ACCOUNTS_PATH = require('../configs/state-manager').statePath('fediverse-accounts.json');
const FETCH_TIMEOUT_MS = 8000;
const TIMELINE_CACHE_MS = 60 * 1000;
const TIMELINE_LIMIT = 40;
@ -18,9 +19,10 @@ const spoofHeaders = (accept) => ({
});
const isPrivateHost = (hostname) => {
const h = String(hostname || '').toLowerCase();
let h = String(hostname || '').toLowerCase().trim().replace(/^\[|\]$/g, '');
if (!h) return true;
if (h === 'localhost' || h.endsWith('.localhost') || h.endsWith('.local')) return true;
if (h.startsWith('::ffff:')) h = h.slice(7);
if (h === 'localhost' || h.endsWith('.localhost') || h.endsWith('.local') || h === '0.0.0.0' || h === '::') return true;
if (h === '::1' || h.startsWith('fe80:') || h.startsWith('fc') || h.startsWith('fd')) return true;
const m = h.match(/^(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})$/);
if (m) {
@ -34,6 +36,89 @@ const isPrivateHost = (hostname) => {
return false;
};
const TG_TIMEOUT_MS = 20000;
const TG_MEDIA_MAX_BYTES = 25 * 1024 * 1024;
const PROXY_MEDIA_MAX_BYTES = TG_MEDIA_MAX_BYTES;
const networkPaused = () => {
if (process.env.OASIS_NETWORK_PAUSED === '1') return true;
try { return require('../configs/config-manager.js').getConfig().networkPaused === true; } catch (_) { return true; }
};
const netFetch = (url, opts) => networkPaused() ? Promise.reject(new Error('Network is paused')) : fetch(url, opts);
const publicOnlyLookup = (hostname, options, callback) => {
require('dns').lookup(hostname, { ...(options || {}), all: true }, (err, addresses) => {
if (err) return callback(err);
const list = Array.isArray(addresses) ? addresses : [];
if (!list.length || list.some(a => isPrivateHost(a.address))) return callback(Object.assign(new Error('Refusing a private address'), { code: 'EPRIVATE' }));
if (options && options.all) return callback(null, list);
callback(null, list[0].address, list[0].family);
});
};
let publicClient;
const publicFetch = (url, opts) => {
if (networkPaused()) return Promise.reject(new Error('Network is paused'));
if (publicClient === undefined) {
try {
const { Agent, fetch: clientFetch } = require('../server/node_modules/undici');
publicClient = { fetch: clientFetch, dispatcher: new Agent({ connect: { lookup: publicOnlyLookup } }) };
} catch (_) { publicClient = null; }
}
return publicClient ? publicClient.fetch(url, { ...opts, dispatcher: publicClient.dispatcher }) : fetch(url, opts);
};
const readCapped = async (res, max) => {
if (Number(res.headers.get('content-length') || 0) > max) return null;
if (!res.body || typeof res.body.getReader !== 'function') {
const ab = await res.arrayBuffer();
return ab.byteLength > max ? null : Buffer.from(ab);
}
const reader = res.body.getReader();
const parts = [];
let total = 0;
for (;;) {
const { done, value } = await reader.read();
if (done) break;
total += value.length;
if (total > max) { try { await reader.cancel(); } catch (_) {} return null; }
parts.push(Buffer.from(value));
}
return Buffer.concat(parts);
};
const TG_MEDIA_CACHE_MAX = 60;
const TG_LOGIN_TTL_MS = 10 * 60 * 1000;
let gram = null;
const loadGram = () => {
if (gram) return gram;
const { TelegramClient, Api } = require('../server/node_modules/telegram');
const { StringSession } = require('../server/node_modules/telegram/sessions');
const { Logger, LogLevel } = require('../server/node_modules/telegram/extensions/Logger');
gram = { TelegramClient, Api, StringSession, Logger, LogLevel };
return gram;
};
const withTimeout = (promise, ms = TG_TIMEOUT_MS) => new Promise((resolve, reject) => {
const timer = setTimeout(() => reject(new Error('telegramErrTimeout')), ms);
Promise.resolve(promise).then((v) => { clearTimeout(timer); resolve(v); }, (e) => { clearTimeout(timer); reject(e); });
});
const tgDisplayName = (u) => {
if (!u) return '';
if (u.title) return String(u.title);
const full = [u.firstName, u.lastName].filter(Boolean).join(' ').trim();
return full || (u.username ? `@${u.username}` : '');
};
const tgErrCode = (err) => {
if (err && err.message === 'peerErrPaused') return 'peerErrPaused';
const m = String((err && (err.errorMessage || err.message)) || '').toUpperCase();
if (m.includes('PHONE_CODE_INVALID') || m.includes('PHONE_CODE_EMPTY')) return 'telegramErrCode';
if (m.includes('PHONE_CODE_EXPIRED')) return 'telegramErrCodeExpired';
if (m.includes('PASSWORD_HASH_INVALID')) return 'telegramErrPassword';
if (m.includes('PHONE_NUMBER_INVALID') || m.includes('PHONE_NUMBER_BANNED')) return 'telegramErrPhone';
if (m.includes('API_ID_INVALID') || m.includes('API_ID_PUBLISHED_FLOOD')) return 'telegramErrApi';
if (m.includes('FLOOD')) return 'telegramErrFlood';
if (m.includes('TELEGRAMERRTIMEOUT')) return 'telegramErrTimeout';
if (m.includes('AUTH_KEY') || m.includes('SESSION_REVOKED') || m.includes('USER_DEACTIVATED') || m.includes('AUTH_USER_CANCEL')) return 'telegramErrAuth';
return 'telegramErrConnect';
};
const tgHardError = (code) => code === 'telegramErrPhone' || code === 'telegramErrApi' || code === 'telegramErrFlood' || code === 'telegramErrAuth' || code === 'telegramErrConnect' || code === 'telegramErrTimeout';
module.exports = ({ isPublic } = {}) => {
const cache = new Map();
let mediaHosts = new Set();
@ -82,7 +167,7 @@ module.exports = ({ isPublic } = {}) => {
const timer = setTimeout(() => controller.abort(), FETCH_TIMEOUT_MS);
try {
const headers = Object.assign(spoofHeaders('application/json'), { Authorization: `Bearer ${token}` }, opts.headers || {});
const res = await fetch(`${instance}${pathName}`, { ...opts, headers, signal: controller.signal });
const res = await netFetch(`${instance}${pathName}`, { ...opts, headers, signal: controller.signal });
return res;
} finally {
clearTimeout(timer);
@ -287,8 +372,15 @@ module.exports = ({ isPublic } = {}) => {
if (!file || !file.filepath) return null;
let buf;
try { buf = fs.readFileSync(file.filepath); } catch (_) { throw new Error('fediverseErrMedia'); }
return api.uploadMediaBuffer({ buffer: buf, mimetype: file.mimetype, filename: file.originalFilename });
},
async uploadMediaBuffer({ buffer, mimetype, filename } = {}) {
const m = getMastodon();
if (!m) throw new Error('fediverseErrAuth');
if (!buffer || !buffer.length) return null;
const fd = new FormData();
fd.append('file', new Blob([buf], { type: file.mimetype || 'application/octet-stream' }), file.originalFilename || 'upload');
fd.append('file', new Blob([buffer], { type: mimetype || 'application/octet-stream' }), filename || 'upload');
let res;
try {
res = await apiFetch(m.instance, m.token, '/api/v2/media', { method: 'POST', body: fd });
@ -361,6 +453,656 @@ module.exports = ({ isPublic } = {}) => {
favourite(id) { return this.action(id, 'favourite'); },
unfavourite(id) { return this.action(id, 'unfavourite'); },
telegram: (() => {
const getTg = () => readStore().telegram || null;
let client = null;
let pending = null;
const entities = new Map();
const messages = new Map();
const mediaCache = new Map();
const avatarCache = new Map();
const dialogCache = { ts: 0, value: null };
const killClient = async (c) => {
if (!c) return;
try { await withTimeout(c.destroy(), 8000); } catch (_) { try { await c.disconnect(); } catch (__) {} }
};
const newClient = (sessionStr, apiId, apiHash) => {
const { TelegramClient, StringSession, Logger, LogLevel } = loadGram();
return new TelegramClient(new StringSession(sessionStr || ''), Number(apiId), String(apiHash), {
connectionRetries: 3,
baseLogger: new Logger(LogLevel.NONE)
});
};
const getClient = async () => {
const t = getTg();
if (!t || !t.session) throw new Error('telegramErrAuth');
if (networkPaused()) {
const c = client;
client = null;
await killClient(c);
throw new Error('peerErrPaused');
}
if (client && client.connected) return client;
if (!client) client = newClient(t.session, t.apiId, t.apiHash);
await withTimeout(client.connect());
return client;
};
const rememberDialogs = (list) => {
for (const d of list) if (d && d.entity) entities.set(String(d.id), d.entity);
};
const resolveEntity = async (c, id) => {
const key = String(id);
if (entities.has(key)) return entities.get(key);
try { rememberDialogs(await withTimeout(c.getDialogs({ limit: TIMELINE_LIMIT }))); } catch (_) {}
if (entities.has(key)) return entities.get(key);
const ent = await withTimeout(c.getEntity(Number(key)));
entities.set(key, ent);
return ent;
};
const peerKind = (d) => (d.isUser ? 'user' : d.isChannel ? 'channel' : 'group');
const kindOfEntity = (ent) => {
const { Api } = loadGram();
if (ent instanceof Api.User) return 'user';
if (ent instanceof Api.Channel) return ent.megagroup ? 'group' : 'channel';
return 'group';
};
const mediaInfo = (msg) => {
const { Api } = loadGram();
const m = msg && msg.media;
if (!m) return null;
if (m instanceof Api.MessageMediaPhoto) return { kind: 'photo', mime: 'image/jpeg', size: 0, name: '' };
if (m instanceof Api.MessageMediaDocument && m.document && m.document.mimeType !== undefined) {
const mime = String(m.document.mimeType || 'application/octet-stream');
const size = Number(m.document.size || 0);
const kind = mime.startsWith('image/') ? 'photo' : mime.startsWith('video/') ? 'video' : mime.startsWith('audio/') ? 'audio' : 'file';
let name = '';
for (const attr of (m.document.attributes || [])) if (attr && attr.fileName) name = String(attr.fileName);
return { kind, mime, size, name };
}
return { kind: 'other', mime: '', size: 0, name: '' };
};
const toIso = (unix) => (unix ? new Date(Number(unix) * 1000).toISOString() : '');
const mapMessage = (msg, chatId, senderNames) => {
const info = mediaInfo(msg);
const sid = msg.senderId ? String(msg.senderId) : '';
return {
id: Number(msg.id),
chatId: String(chatId),
text: String(msg.message || ''),
date: toIso(msg.date),
out: msg.out === true,
senderId: sid,
senderName: senderNames.get(sid) || '',
media: info && info.kind !== 'other' && info.size <= TG_MEDIA_MAX_BYTES
? { ...info, url: `/fediverse/telegram/media/${encodeURIComponent(String(chatId))}/${encodeURIComponent(String(msg.id))}` }
: null
};
};
const trimCache = (map) => {
while (map.size > TG_MEDIA_CACHE_MAX) map.delete(map.keys().next().value);
};
const dropPending = async () => {
if (pending && pending.client && pending.client !== client) await killClient(pending.client);
pending = null;
};
const waitStep = async (p, prevStep, ms = 25000) => {
const started = Date.now();
while (Date.now() - started < ms) {
if (p.done || p.error || p.step !== prevStep || p.awaiting) return;
await sleep(200);
}
};
return {
hasAccount() { return !!(getTg() && getTg().session); },
getAccount() {
const t = getTg();
if (!t || !t.session) return null;
return {
network: 'telegram',
username: t.username || '',
displayName: t.displayName || t.username || t.phone || '',
phone: t.phone || '',
avatar: '/fediverse/telegram/avatar/me'
};
},
async getAccountStats() {
if (!this.hasAccount()) return null;
const data = await this.getDialogs();
if (!data || data.error) return null;
return {
chats: data.dialogs.length,
unread: data.dialogs.reduce((sum, d) => sum + (Number(d.unread) || 0), 0)
};
},
loginState() {
if (!pending) return null;
if (Date.now() - pending.startedAt > TG_LOGIN_TTL_MS && !pending.done) { dropPending(); return null; }
return { step: pending.step, error: pending.error || '', done: pending.done === true };
},
async beginLogin({ apiId, apiHash, phone }) {
if (isPublic) throw new Error('fediverseErrPublic');
const id = Number(String(apiId || '').trim());
const hash = String(apiHash || '').trim();
const ph = String(phone || '').trim();
if (!id || !hash || !ph) throw new Error('telegramErrMissing');
await dropPending();
if (networkPaused()) throw new Error('peerErrPaused');
const c = newClient('', id, hash);
try { await withTimeout(c.connect()); } catch (err) { throw new Error(tgErrCode(err)); }
const p = { client: c, apiId: id, apiHash: hash, phone: ph, step: 'sending', error: '', done: false, awaiting: false, startedAt: Date.now(), resolveCode: null, resolvePassword: null };
pending = p;
p.promise = c.start({
phoneNumber: async () => ph,
phoneCode: async () => new Promise((resolve) => { p.step = 'code'; p.awaiting = true; p.resolveCode = resolve; }),
password: async () => new Promise((resolve) => { p.step = 'password'; p.awaiting = true; p.resolvePassword = resolve; }),
onError: (err) => {
const code = tgErrCode(err);
p.error = code;
return tgHardError(code);
}
}).then(async () => {
const me = await withTimeout(c.getMe());
const store = readStore();
store.telegram = {
apiId: id,
apiHash: hash,
phone: ph,
session: c.session.save(),
userId: String(me.id || ''),
username: me.username || '',
displayName: tgDisplayName(me)
};
writeStore(store);
if (client && client !== c) await killClient(client);
client = c;
entities.clear(); dialogCache.value = null;
p.done = true; p.step = 'done'; p.error = '';
pending = null;
setModuleFlag('on');
}).catch(async (err) => {
if (!p.error) p.error = tgErrCode(err);
p.done = true; p.step = 'error';
await killClient(c);
});
await waitStep(p, 'sending', 12000);
return this.loginState();
},
async submitCode(code) {
const p = pending;
if (!p || p.done || typeof p.resolveCode !== 'function') throw new Error('telegramErrNoLogin');
const value = String(code || '').replace(/\s+/g, '');
if (!value) throw new Error('telegramErrCode');
const resolve = p.resolveCode;
p.resolveCode = null; p.awaiting = false; p.error = '';
resolve(value);
await waitStep(p, 'code');
return this.loginState();
},
async submitPassword(password) {
const p = pending;
if (!p || p.done || typeof p.resolvePassword !== 'function') throw new Error('telegramErrNoLogin');
const value = String(password || '');
if (!value) throw new Error('telegramErrPassword');
const resolve = p.resolvePassword;
p.resolvePassword = null; p.awaiting = false; p.error = '';
resolve(value);
await waitStep(p, 'password');
return this.loginState();
},
async cancelLogin() { await dropPending(); },
async goOffline() {
await dropPending();
const c = client;
client = null;
await killClient(c);
},
async disconnect() {
const { Api } = loadGram();
try {
const c = await getClient();
await withTimeout(c.invoke(new Api.auth.LogOut()), 8000);
} catch (_) {}
await killClient(client);
client = null;
const store = readStore();
delete store.telegram;
writeStore(store);
entities.clear(); messages.clear(); mediaCache.clear(); avatarCache.clear(); dialogCache.value = null;
return true;
},
invalidateCache() { dialogCache.value = null; },
async getDialogs(force = false) {
if (!this.hasAccount()) return { connected: false, dialogs: [] };
const now = Date.now();
if (!force && dialogCache.value && now - dialogCache.ts < TIMELINE_CACHE_MS) return dialogCache.value;
let c;
try { c = await getClient(); } catch (err) { return { connected: true, account: this.getAccount(), dialogs: [], error: tgErrCode(err) }; }
let list;
try { list = await withTimeout(c.getDialogs({ limit: TIMELINE_LIMIT })); } catch (err) { return { connected: true, account: this.getAccount(), dialogs: [], error: tgErrCode(err) }; }
rememberDialogs(list);
const dialogs = list.map((d) => {
const id = String(d.id);
const last = d.message || null;
return {
id,
title: String(d.title || d.name || id),
kind: peerKind(d),
unread: Number(d.unreadCount || 0),
date: toIso(d.date || (last && last.date)),
last: last ? { text: String(last.message || ''), out: last.out === true, hasMedia: !!last.media } : null,
avatar: `/fediverse/telegram/avatar/${encodeURIComponent(id)}`
};
});
const value = { connected: true, account: this.getAccount(), dialogs };
dialogCache.ts = now; dialogCache.value = value;
return value;
},
async getChat(id) {
const c = await getClient();
const entity = await resolveEntity(c, id);
const list = await withTimeout(c.getMessages(entity, { limit: TIMELINE_LIMIT }));
const senderNames = new Map();
for (const msg of list) {
const sid = msg.senderId ? String(msg.senderId) : '';
if (!sid || senderNames.has(sid)) continue;
try { senderNames.set(sid, tgDisplayName(await withTimeout(msg.getSender(), 8000))); } catch (_) { senderNames.set(sid, ''); }
}
for (const msg of list) messages.set(`${id}:${msg.id}`, msg);
trimCache(messages);
try { await withTimeout(c.markAsRead(entity), 8000); } catch (_) {}
dialogCache.value = null;
return {
id: String(id),
title: tgDisplayName(entity) || String(id),
kind: kindOfEntity(entity),
messages: list.slice().reverse().map((msg) => mapMessage(msg, id, senderNames))
};
},
async sendMessage(id, { text, file } = {}) {
const c = await getClient();
const entity = await resolveEntity(c, id);
const msg = String(text || '').trim();
if (file && file.filepath) {
await withTimeout(c.sendFile(entity, { file: file.filepath, caption: msg, forceDocument: false }), 90000);
} else {
if (!msg) throw new Error('fediverseErrEmpty');
await withTimeout(c.sendMessage(entity, { message: msg }));
}
dialogCache.value = null;
return true;
},
async getMedia(chatId, msgId) {
const key = `${chatId}:${msgId}`;
if (mediaCache.has(key)) return mediaCache.get(key);
const c = await getClient();
let msg = messages.get(key) || null;
if (!msg) {
const entity = await resolveEntity(c, chatId);
const arr = await withTimeout(c.getMessages(entity, { ids: [Number(msgId)] }));
msg = arr && arr[0] ? arr[0] : null;
}
if (!msg || !msg.media) return null;
const info = mediaInfo(msg);
if (!info || info.kind === 'other' || info.size > TG_MEDIA_MAX_BYTES) return null;
const buf = await withTimeout(c.downloadMedia(msg, {}), 90000);
if (!buf || !buf.length) return null;
const out = { contentType: info.mime || 'application/octet-stream', buffer: Buffer.from(buf), name: info.name };
mediaCache.set(key, out);
trimCache(mediaCache);
return out;
},
async getAvatar(id) {
const key = String(id);
if (avatarCache.has(key)) return avatarCache.get(key);
const c = await getClient();
const entity = key === 'me' ? 'me' : await resolveEntity(c, key);
let buf = null;
try { buf = await withTimeout(c.downloadProfilePhoto(entity, { isBig: false }), 15000); } catch (_) { buf = null; }
const out = buf && buf.length ? { contentType: 'image/jpeg', buffer: Buffer.from(buf) } : null;
avatarCache.set(key, out);
trimCache(avatarCache);
return out;
}
};
})(),
peertube: (() => {
const getPt = () => readStore().peertube || null;
const feedCache = new Map();
const PT_UPLOAD_TIMEOUT_MS = 15 * 60 * 1000;
const PT_PRIVACY = { public: 1, unlisted: 2, private: 3 };
const fetchTimed = async (url, opts = {}, timeoutMs = FETCH_TIMEOUT_MS) => {
const controller = new AbortController();
const timer = setTimeout(() => controller.abort(), timeoutMs);
try { return await netFetch(url, { ...opts, signal: controller.signal }); } finally { clearTimeout(timer); }
};
const oauthClient = async (instance) => {
let res;
try { res = await fetchTimed(`${instance}/api/v1/oauth-clients/local`, { headers: spoofHeaders('application/json') }); } catch (_) { throw new Error('peertubeErrConnect'); }
if (!res.ok) throw new Error('peertubeErrConnect');
const data = await res.json().catch(() => null);
if (!data || !data.client_id || !data.client_secret) throw new Error('peertubeErrConnect');
return data;
};
const tokenRequest = async (instance, params) => {
let res;
try {
res = await fetchTimed(`${instance}/api/v1/users/token`, {
method: 'POST',
headers: { ...spoofHeaders('application/json'), 'Content-Type': 'application/x-www-form-urlencoded' },
body: new URLSearchParams(params).toString()
});
} catch (_) { throw new Error('peertubeErrConnect'); }
if (res.status === 400 || res.status === 401 || res.status === 403) throw new Error('peertubeErrAuth');
if (!res.ok) throw new Error('peertubeErrConnect');
const data = await res.json().catch(() => null);
if (!data || !data.access_token) throw new Error('peertubeErrAuth');
return data;
};
const saveTokens = (pt, data) => {
const store = readStore();
if (!store.peertube) return null;
store.peertube = { ...store.peertube, token: data.access_token, refreshToken: data.refresh_token || pt.refreshToken || '', expiresAt: Date.now() + (Number(data.expires_in) || 0) * 1000 };
writeStore(store);
return store.peertube;
};
const refreshAccess = async (pt) => {
if (!pt || !pt.refreshToken) return null;
try {
const data = await tokenRequest(pt.instance, { client_id: pt.clientId, client_secret: pt.clientSecret, grant_type: 'refresh_token', refresh_token: pt.refreshToken });
return saveTokens(pt, data);
} catch (_) { return null; }
};
const ptFetch = async (pathName, opts = {}, retry = true) => {
let pt = getPt();
if (!pt) throw new Error('peertubeErrAuth');
if (pt.expiresAt && Date.now() > pt.expiresAt - 60000 && retry) { const fresh = await refreshAccess(pt); if (fresh) pt = fresh; }
const headers = Object.assign(spoofHeaders('application/json'), { Authorization: `Bearer ${pt.token}` }, opts.headers || {});
let res;
try { res = await fetchTimed(`${pt.instance}${pathName}`, { method: opts.method || 'GET', body: opts.body, headers }, opts.timeoutMs || FETCH_TIMEOUT_MS); } catch (_) { throw new Error('peertubeErrConnect'); }
if (res.status === 401 && retry) {
const fresh = await refreshAccess(pt);
if (fresh) return ptFetch(pathName, opts, false);
}
return res;
};
const absUrl = (pt, p) => {
if (typeof p !== 'string' || !p) return '';
if (/^https?:\/\//i.test(p)) return p;
return `${pt.instance}${p.startsWith('/') ? '' : '/'}${p}`;
};
const rememberHost = (url) => { try { mediaHosts.add(new URL(url).host); } catch (_) {} };
const avatarOf = (pt, obj) => {
const list = Array.isArray(obj && obj.avatars) ? obj.avatars.slice().sort((a, b) => (Number(b.width) || 0) - (Number(a.width) || 0)) : [];
const p = list.length ? list[0].path : (obj && obj.avatar && obj.avatar.path) || '';
return p ? proxify(absUrl(pt, p)) : '';
};
const channelOf = (pt, ch) => {
if (!ch || typeof ch !== 'object') return { name: '', displayName: '', host: '', handle: '', avatar: '', url: '' };
const host = ch.host || (() => { try { return new URL(pt.instance).host; } catch (_) { return ''; } })();
return { name: ch.name || '', displayName: ch.displayName || ch.name || '', host, handle: ch.name ? `${ch.name}@${host}` : '', avatar: avatarOf(pt, ch), url: ch.url || (ch.name ? `${pt.instance}/c/${ch.name}` : '') };
};
const mapVideo = (pt, v) => {
if (!v || typeof v !== 'object') return null;
const id = String(v.uuid || v.shortUUID || v.id || '');
return {
id,
numericId: v.id,
name: typeof v.name === 'string' ? v.name : '',
description: typeof v.description === 'string' ? v.description : '',
thumbnail: proxify(absUrl(pt, v.thumbnailPath || v.previewPath || '')),
duration: Number(v.duration) || 0,
views: Number(v.views) || 0,
likes: Number(v.likes) || 0,
dislikes: Number(v.dislikes) || 0,
publishedAt: v.publishedAt || v.createdAt || '',
channel: channelOf(pt, v.channel),
url: typeof v.url === 'string' && v.url ? v.url : `${pt.instance}/w/${id}`,
privacy: v.privacy && v.privacy.label ? String(v.privacy.label) : '',
privacyId: v.privacy && v.privacy.id ? Number(v.privacy.id) : 0,
state: v.state && v.state.label ? String(v.state.label) : '',
isLive: v.isLive === true,
nsfw: v.nsfw === true
};
};
const pickFile = (v) => {
const candidates = [];
for (const f of (Array.isArray(v.files) ? v.files : [])) if (f && f.fileUrl) candidates.push(f);
for (const pl of (Array.isArray(v.streamingPlaylists) ? v.streamingPlaylists : [])) for (const f of (Array.isArray(pl.files) ? pl.files : [])) if (f && f.fileUrl) candidates.push(f);
if (!candidates.length) return null;
const resOf = (f) => Number(f.resolution && f.resolution.id) || 0;
const under = candidates.filter(f => resOf(f) > 0 && resOf(f) <= 720).sort((a, b) => resOf(b) - resOf(a));
const pick = under[0] || candidates.slice().sort((a, b) => resOf(a) - resOf(b))[0];
return { url: pick.fileUrl, resolution: resOf(pick), size: Number(pick.size) || 0 };
};
const mapComment = (pt, c) => c && typeof c === 'object' ? {
id: String(c.id || ''),
html: typeof c.text === 'string' ? c.text : '',
createdAt: c.createdAt || '',
totalReplies: Number(c.totalReplies) || 0,
account: c.account ? { name: c.account.name || '', displayName: c.account.displayName || c.account.name || '', host: c.account.host || '', avatar: avatarOf(pt, c.account) } : { name: '', displayName: '', host: '', avatar: '' }
} : null;
const listOf = async (pathName) => {
const res = await ptFetch(pathName);
if (res.status === 401 || res.status === 403) return { error: 'peertubeErrAuth' };
if (!res.ok) return { error: 'peertubeErrFetch' };
const data = await res.json().catch(() => null);
return { total: Number(data && data.total) || 0, data: Array.isArray(data && data.data) ? data.data : [] };
};
const invalidatePt = () => feedCache.clear();
return {
hasAccount() { return !!(getPt() && getPt().token); },
getAccount() {
const pt = getPt();
if (!pt || !pt.token) return null;
let host = '';
try { host = new URL(pt.instance).host; } catch (_) {}
return {
network: 'peertube',
instance: pt.instance,
host,
username: pt.username || '',
displayName: pt.displayName || pt.username || '',
handle: pt.username ? `${pt.username}@${host}` : host,
avatar: pt.avatar ? proxify(pt.avatar) : '',
channel: pt.channelName || '',
channelDisplayName: pt.channelDisplayName || pt.channelName || '',
channelUrl: pt.channelName ? `${pt.instance}/c/${pt.channelName}` : pt.instance,
profileUrl: pt.username ? `${pt.instance}/a/${pt.username}` : pt.instance
};
},
async connect({ instance, username, password }) {
if (isPublic) throw new Error('fediverseErrPublic');
const origin = (() => { try { return normalizeInstance(instance); } catch (_) { throw new Error('peertubeErrInstance'); } })();
const user = String(username || '').trim();
const pass = String(password || '');
if (!user || !pass) throw new Error('peertubeErrMissing');
const client = await oauthClient(origin);
const tokens = await tokenRequest(origin, { client_id: client.client_id, client_secret: client.client_secret, grant_type: 'password', response_type: 'code', username: user, password: pass });
let me;
try {
const res = await fetchTimed(`${origin}/api/v1/users/me`, { headers: Object.assign(spoofHeaders('application/json'), { Authorization: `Bearer ${tokens.access_token}` }) });
if (!res.ok) throw new Error('peertubeErrAuth');
me = await res.json();
} catch (_) { throw new Error('peertubeErrConnect'); }
const channel = Array.isArray(me.videoChannels) && me.videoChannels.length ? me.videoChannels[0] : null;
const acc = me.account || {};
const avatars = Array.isArray(acc.avatars) ? acc.avatars.slice().sort((a, b) => (Number(b.width) || 0) - (Number(a.width) || 0)) : [];
const avatarPath = avatars.length ? avatars[0].path : (acc.avatar && acc.avatar.path) || '';
const store = readStore();
store.peertube = {
instance: origin,
clientId: client.client_id,
clientSecret: client.client_secret,
token: tokens.access_token,
refreshToken: tokens.refresh_token || '',
expiresAt: Date.now() + (Number(tokens.expires_in) || 0) * 1000,
userId: String(me.id || ''),
username: me.username || user,
displayName: acc.displayName || me.username || user,
avatar: avatarPath ? `${origin}${avatarPath.startsWith('/') ? '' : '/'}${avatarPath}` : '',
accountName: acc.name || me.username || user,
channelId: channel ? String(channel.id) : '',
channelName: channel ? channel.name || '' : '',
channelDisplayName: channel ? channel.displayName || channel.name || '' : ''
};
writeStore(store);
invalidatePt();
try { mediaHosts.add(new URL(origin).host); } catch (_) {}
setModuleFlag('on');
return this.getAccount();
},
async disconnect() {
const pt = getPt();
if (pt && pt.token) {
try { await fetchTimed(`${pt.instance}/api/v1/users/revoke-token`, { method: 'POST', headers: Object.assign(spoofHeaders('application/json'), { Authorization: `Bearer ${pt.token}` }) }); } catch (_) {}
}
const store = readStore();
delete store.peertube;
writeStore(store);
invalidatePt();
return true;
},
invalidateCache() { invalidatePt(); },
async getAccountStats() {
const pt = getPt();
if (!pt) return null;
let me = null;
try { const res = await ptFetch('/api/v1/users/me'); if (res.ok) me = await res.json().catch(() => null); } catch (_) {}
if (!me) return null;
const channel = Array.isArray(me.videoChannels) && me.videoChannels.length ? me.videoChannels[0] : null;
let videos = 0, subscriptions = 0;
try { const v = await listOf('/api/v1/users/me/videos?start=0&count=1'); if (!v.error) videos = v.total; } catch (_) {}
try { const sub = await listOf('/api/v1/users/me/subscriptions?start=0&count=1'); if (!sub.error) subscriptions = sub.total; } catch (_) {}
return {
videos,
subscriptions,
followers: channel ? Number(channel.followersCount) || 0 : 0,
quota: Number(me.videoQuota) || 0,
quotaUsed: Number(me.videoQuotaUsed) || 0
};
},
async getFeed(kind = 'subscriptions', force = false) {
const pt = getPt();
if (!pt) return { connected: false, videos: [] };
try { mediaHosts.add(new URL(pt.instance).host); } catch (_) {}
const key = kind === 'mine' ? 'mine' : 'subscriptions';
const now = Date.now();
const hit = feedCache.get(key);
if (!force && hit && now - hit.ts < TIMELINE_CACHE_MS) return hit.value;
let result;
try {
result = await listOf(key === 'mine' ? `/api/v1/users/me/videos?start=0&count=${TIMELINE_LIMIT}&sort=-publishedAt` : `/api/v1/users/me/subscriptions/videos?start=0&count=${TIMELINE_LIMIT}&sort=-publishedAt`);
} catch (err) {
return { connected: true, account: this.getAccount(), videos: [], error: (err && err.message) || 'peertubeErrFetch' };
}
if (result.error) return { connected: true, account: this.getAccount(), videos: [], error: result.error };
const value = { connected: true, account: this.getAccount(), videos: result.data.map(v => mapVideo(pt, v)).filter(Boolean), total: result.total, kind: key };
feedCache.set(key, { ts: now, value });
return value;
},
async getVideo(id) {
const pt = getPt();
if (!pt) return null;
const vid = encodeURIComponent(String(id || ''));
let res;
try { res = await ptFetch(`/api/v1/videos/${vid}`); } catch (_) { return { error: 'peertubeErrFetch' }; }
if (!res.ok) return { error: 'peertubeErrFetch' };
const raw = await res.json().catch(() => null);
const video = mapVideo(pt, raw);
if (!video) return { error: 'peertubeErrFetch' };
const file = pickFile(raw || {});
if (file) { rememberHost(file.url); video.stream = `/fediverse/peertube/stream?u=${encodeURIComponent(file.url)}`; video.resolution = file.resolution; video.fileSize = file.size; }
video.embedUrl = raw && raw.embedPath ? absUrl(pt, raw.embedPath) : '';
video.tags = Array.isArray(raw && raw.tags) ? raw.tags.map(String) : [];
let comments = [];
try {
const c = await listOf(`/api/v1/videos/${vid}/comment-threads?start=0&count=${TIMELINE_LIMIT}&sort=-createdAt`);
if (!c.error) comments = c.data.map(x => mapComment(pt, x)).filter(Boolean);
} catch (_) {}
let myRating = 'none';
try { const r = await ptFetch(`/api/v1/users/me/videos/${vid}/rating`); if (r.ok) { const d = await r.json().catch(() => null); if (d && d.rating) myRating = String(d.rating); } } catch (_) {}
return { video, comments, myRating };
},
async rate(id, rating) {
const allowed = { like: 1, dislike: 1, none: 1 };
if (!allowed[rating]) throw new Error('peertubeErrFetch');
let res;
try { res = await ptFetch(`/api/v1/videos/${encodeURIComponent(String(id || ''))}/rate`, { method: 'PUT', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ rating }) }); } catch (_) { throw new Error('peertubeErrConnect'); }
if (res.status === 401 || res.status === 403) throw new Error('peertubeErrAuth');
if (!res.ok && res.status !== 204) throw new Error('peertubeErrFetch');
invalidatePt();
return true;
},
async comment(id, text) {
const body = String(text || '').trim();
if (!body) throw new Error('fediverseErrEmpty');
let res;
try { res = await ptFetch(`/api/v1/videos/${encodeURIComponent(String(id || ''))}/comment-threads`, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ text: body }) }); } catch (_) { throw new Error('peertubeErrConnect'); }
if (res.status === 401 || res.status === 403) throw new Error('peertubeErrAuth');
if (!res.ok) throw new Error('peertubeErrComment');
return true;
},
async upload({ file, name, description, privacy } = {}) {
const pt = getPt();
if (!pt) throw new Error('peertubeErrAuth');
if (!file || !file.filepath || !(Number(file.size) > 0)) throw new Error('peertubeErrNoFile');
const title = String(name || '').trim();
if (!title) throw new Error('peertubeErrTitle');
let buf;
try { buf = fs.readFileSync(file.filepath); } catch (_) { throw new Error('peertubeErrUpload'); }
const fd = new FormData();
fd.append('videofile', new Blob([buf], { type: file.mimetype || 'video/mp4' }), file.originalFilename || 'video.mp4');
if (pt.channelId) fd.append('channelId', String(pt.channelId));
fd.append('name', title.slice(0, 120));
fd.append('privacy', String(PT_PRIVACY[String(privacy || 'public')] || 1));
const desc = String(description || '').trim();
if (desc) fd.append('description', desc.slice(0, 10000));
let res;
try { res = await ptFetch('/api/v1/videos/upload', { method: 'POST', body: fd, timeoutMs: PT_UPLOAD_TIMEOUT_MS }); } catch (_) { throw new Error('peertubeErrUpload'); }
if (res.status === 401 || res.status === 403) throw new Error('peertubeErrAuth');
if (!res.ok) throw new Error('peertubeErrUpload');
const data = await res.json().catch(() => ({}));
invalidatePt();
const v = data && data.video ? data.video : {};
return { id: String(v.uuid || v.shortUUID || v.id || '') };
},
async openStream(remoteUrl, range) {
let u;
try { u = new URL(String(remoteUrl || '')); } catch (_) { return null; }
if (u.protocol !== 'https:' || isPrivateHost(u.hostname) || !api.isHostAllowed(u.host)) return null;
const headers = spoofHeaders('*/*');
if (range) headers.Range = String(range);
let res = null;
let current = u;
for (let hop = 0; hop < 5; hop++) {
try { res = await publicFetch(current.href, { headers, redirect: 'manual' }); } catch (_) { return null; }
if (![301, 302, 303, 307, 308].includes(res.status)) break;
let next;
try { next = new URL(res.headers.get('location') || '', current); } catch (_) { return null; }
if (next.protocol !== 'https:' || isPrivateHost(next.hostname)) return null;
current = next;
res = null;
}
if (!res || (!res.ok && res.status !== 206)) return null;
const type = res.headers.get('content-type') || 'application/octet-stream';
if (!/^(video|audio|application\/octet-stream)/i.test(type)) return null;
return {
status: res.status,
type,
length: res.headers.get('content-length') || '',
contentRange: res.headers.get('content-range') || '',
body: res.body ? Readable.fromWeb(res.body) : null
};
}
};
})(),
isHostAllowed(host) {
if (mediaHosts.has(host)) return true;
const m = getMastodon();
@ -368,6 +1110,10 @@ module.exports = ({ isPublic } = {}) => {
try { if (new URL(m.instance).host === host) return true; } catch (_) {}
try { if (m.avatar && new URL(m.avatar).host === host) return true; } catch (_) {}
}
const pt = readStore().peertube;
if (pt && pt.instance) {
try { if (new URL(pt.instance).host === host) return true; } catch (_) {}
}
return false;
},
@ -382,7 +1128,7 @@ module.exports = ({ isPublic } = {}) => {
try {
let current = u, res = null;
for (let hop = 0; hop < 5; hop++) {
res = await fetch(current.href, { signal: controller.signal, redirect: 'manual', headers: spoofHeaders('image/avif,image/webp,*/*') });
res = await publicFetch(current.href, { signal: controller.signal, redirect: 'manual', headers: spoofHeaders('image/avif,image/webp,*/*') });
if (res.status < 300 || res.status >= 400) break;
const loc = res.headers.get('location');
if (!loc) return null;
@ -395,8 +1141,8 @@ module.exports = ({ isPublic } = {}) => {
if (!res || !res.ok) return null;
const ct = res.headers.get('content-type') || 'application/octet-stream';
if (!/^(image|video|audio)\//i.test(ct)) return null;
const ab = await res.arrayBuffer();
return { contentType: ct, buffer: Buffer.from(ab) };
const buffer = await readCapped(res, PROXY_MEDIA_MAX_BYTES);
return buffer ? { contentType: ct, buffer } : null;
} catch (_) {
return null;
} finally {
@ -407,3 +1153,5 @@ module.exports = ({ isPublic } = {}) => {
return api;
};
module.exports.isPrivateHost = isPrivateHost;

93
src/models/feed_model.js
View file

@ -2,7 +2,8 @@ const pull = require("../server/node_modules/pull-stream");
const { getConfig } = require("../configs/config-manager.js");
const categories = require("../backend/opinion_categories");
const { buildValidatedTombstoneSet } = require('./tombstone_validator');
const { dedupeBy, mergeDuplicatesBy, norm } = require('../backend/dedupe');
const { readTyped } = require('./typed_log');
const { dedupeBy, mergeDuplicatesBy, norm, versionGroups, versionEntry } = require('../backend/dedupe');
const logLimit = getConfig().ssbLogStream?.limit || 1000;
const FEED_TEXT_MIN = Number(getConfig().feed?.minLength ?? 1);
@ -15,10 +16,10 @@ module.exports = ({ cooler }) => {
return ssb;
};
const cleanText = (t) => (typeof t === "string" ? t.trim() : "");
const cleanText = (t) => (typeof t === "string" ? t.replace(/\r\n?/g, "\n").trim() : "");
const isValidFeedText = (t) => {
const s = cleanText(t);
const s = cleanText(t).replace(/!?\[[^\]\n]*\]\(\s*&[^)\s]+\.sha256\s*\)/g, "").trim();
return s.length >= FEED_TEXT_MIN && s.length <= FEED_TEXT_MAX;
};
@ -27,10 +28,9 @@ module.exports = ({ cooler }) => {
ssbClient.get(id, (err, val) => (err ? reject(err) : resolve({ key: id, value: val })));
});
const getAllMessages = (ssbClient) =>
new Promise((resolve, reject) => {
pull(ssbClient.createLogStream({ limit: logLimit }), pull.collect((err, msgs) => (err ? reject(err) : resolve(msgs))));
});
const FEED_TYPES = ["feed", "feedOpinion", "feed-action", "tombstone"];
const getAllMessages = async (ssbClient) => readTyped(ssbClient, FEED_TYPES, { limit: logLimit });
const extractTags = (text) => {
const list = (String(text || "").match(/#[A-Za-z0-9_]{1,32}/g) || []).map((t) => t.slice(1).toLowerCase());
@ -40,11 +40,8 @@ module.exports = ({ cooler }) => {
const buildIndex = async (ssbClient) => {
const messages = await getAllMessages(ssbClient);
const forward = new Map();
const replacedIds = new Set();
const tombstoned = buildValidatedTombstoneSet(messages);
const feedsById = new Map();
const authorById = new Map();
const actions = [];
const opinionMsgs = [];
@ -59,7 +56,6 @@ module.exports = ({ cooler }) => {
}
if (c.type === "feed") {
feedsById.set(k, msg);
authorById.set(k, msg?.value?.author);
continue;
}
if (c.type === "feed-action") {
@ -68,24 +64,8 @@ module.exports = ({ cooler }) => {
}
}
for (const [k, msg] of feedsById) {
const t = msg?.value?.content?.replaces;
if (!t) continue;
if (authorById.get(t) === authorById.get(k)) {
forward.set(t, k);
replacedIds.add(t);
}
}
const resolve = (id) => {
let cur = id;
const seen = new Set();
while (forward.has(cur) && !seen.has(cur)) {
seen.add(cur);
cur = forward.get(cur);
}
return cur;
};
const vg = versionGroups(Array.from(feedsById.values()).map(versionEntry));
const resolve = (id) => vg.tipOf(id);
const actionsByRoot = new Map();
for (const a of actions) {
@ -105,7 +85,7 @@ module.exports = ({ cooler }) => {
opinionsByTip.get(tip).push(op);
}
return { resolve, tombstoned, feedsById, replacedIds, actionsByRoot, opinionsByTip };
return { resolve, vg, tombstoned, feedsById, actionsByRoot, opinionsByTip };
};
const resolveCurrentId = async (id) => {
@ -114,7 +94,7 @@ module.exports = ({ cooler }) => {
return idx.resolve(id);
};
const createFeed = async (text, mentions) => {
const createFeed = async (text, mentions, media = null) => {
const ssbClient = await openSsb();
const userId = ssbClient.id;
@ -129,7 +109,7 @@ module.exports = ({ cooler }) => {
const content = {
type: "feed",
text: cleaned,
text: typeof media === "string" && media.trim() ? `${cleaned}\n${media.trim()}` : cleaned,
author: userId,
createdAt: new Date().toISOString(),
tags: extractTags(cleaned),
@ -146,6 +126,7 @@ module.exports = ({ cooler }) => {
const userId = ssbClient.id;
const idx = await buildIndex(ssbClient);
if (idx.vg.isForeignCopy(idx.vg.rootOf(contentId))) throw new Error("Invalid feed");
const tipId = idx.resolve(contentId);
let msg;
@ -216,6 +197,7 @@ module.exports = ({ cooler }) => {
const userId = ssbClient.id;
const idx = await buildIndex(ssbClient);
if (idx.vg.isForeignCopy(idx.vg.rootOf(contentId))) throw new Error("Invalid feed");
const tipId = idx.resolve(contentId);
const msg = idx.feedsById.get(tipId);
@ -249,30 +231,22 @@ module.exports = ({ cooler }) => {
return !!c && c.type === "feed" && isValidFeedText(c.text);
};
let tips = Array.from(idx.feedsById.values()).filter(
let tips = Array.from(idx.vg.roots).map((r) => idx.feedsById.get(idx.resolve(r))).filter(
(m) =>
!idx.replacedIds.has(m.key) &&
!!m &&
!idx.tombstoned.has(m.key) &&
isValidFeedMsg(m)
);
const textEditedEver = (m) => {
const seen = new Set();
let cur = m;
let lastText = cur?.value?.content?.text;
while (cur?.value?.content?.replaces) {
const prevId = cur.value.content.replaces;
if (!prevId || seen.has(prevId)) break;
seen.add(prevId);
const prev = idx.feedsById.get(prevId);
if (!prev) break;
if (prev?.value?.author !== cur?.value?.author) break;
const prevText = prev?.value?.content?.text;
if (typeof lastText === "string" && typeof prevText === "string" && lastText !== prevText) return true;
cur = prev;
lastText = prevText;
const root = idx.vg.rootOf(m.key);
const owner = idx.feedsById.get(root)?.value?.author;
const texts = new Set();
for (const k of (idx.vg.groups.get(root) || [])) {
const v = idx.feedsById.get(k)?.value;
if (v && v.author === owner && typeof v.content?.text === "string") texts.add(v.content.text);
}
return false;
return texts.size > 1;
};
const materialize = (feedMsg) => {
@ -300,7 +274,7 @@ module.exports = ({ cooler }) => {
}
if (ac.action === "comment") {
commentCount++;
if (!idx.tombstoned.has(a.key)) commentCount++;
continue;
}
}
@ -367,7 +341,7 @@ module.exports = ({ cooler }) => {
if (filter === "MINE") {
feeds = feeds.filter((m) => (m.value?.author || m.value?.content?.author) === userId);
} else if (filter === "TODAY") {
} else if (filter === "RECENT" || filter === "TODAY") {
feeds = feeds.filter((m) => now - getTs(m) < 86400000);
}
@ -388,7 +362,7 @@ module.exports = ({ cooler }) => {
const ssbClient = await openSsb();
const idx = await buildIndex(ssbClient);
const currentId = idx.resolve(feedId);
if (idx.tombstoned.has(currentId)) return null;
if (idx.tombstoned.has(currentId) || idx.vg.isForeignCopy(idx.vg.rootOf(currentId))) return null;
const msg = idx.feedsById.get(currentId);
if (!msg) return null;
const actions = idx.actionsByRoot.get(currentId) || [];
@ -402,7 +376,7 @@ module.exports = ({ cooler }) => {
refeeds++;
if (ac.author || a?.value?.author) refeedsInhabitants.push(ac.author || a.value.author);
}
if (ac.type === "feed-action" && ac.action === "comment") {
if (ac.type === "feed-action" && ac.action === "comment" && !idx.tombstoned.has(a.key)) {
commentCount++;
}
}
@ -417,7 +391,7 @@ module.exports = ({ cooler }) => {
const currentId = idx.resolve(feedId);
const actions = idx.actionsByRoot.get(currentId) || [];
return actions
.filter(a => a?.value?.content?.type === "feed-action" && a?.value?.content?.action === "comment")
.filter(a => a?.value?.content?.type === "feed-action" && a?.value?.content?.action === "comment" && !idx.tombstoned.has(a.key))
.sort((a, b) => (a?.value?.timestamp || 0) - (b?.value?.timestamp || 0));
};
@ -430,6 +404,15 @@ module.exports = ({ cooler }) => {
});
};
return { createFeed, createRefeed, addOpinion, listFeeds, resolveCurrentId, getFeedById, getComments, addComment };
const deleteFeedById = async (id) => {
const ssbClient = await openSsb();
const userId = ssbClient.id;
const currentId = (await resolveCurrentId(id)) || id;
const item = await new Promise((res, rej) => ssbClient.get(currentId, (e, m) => (e || !m || !m.content) ? rej(new Error("Feed not found")) : res(m)));
if (item.author !== userId && item.content?.author !== userId) throw new Error("Not the author");
return new Promise((res, rej) => ssbClient.publish({ type: "tombstone", target: currentId, deletedAt: new Date().toISOString(), author: userId }, (e, m) => e ? rej(e) : res(m)));
};
return { createFeed, createRefeed, addOpinion, listFeeds, resolveCurrentId, getFeedById, getComments, addComment, deleteFeedById };
};

387
src/models/files_model.js Normal file
View file

@ -0,0 +1,387 @@
const pull = require("../server/node_modules/pull-stream");
const { getConfig } = require("../configs/config-manager.js");
const categories = require("../backend/opinion_categories");
const { readTyped, memoIndex } = require("./typed_log");
const { dedupeBy, norm, versionGroups } = require('../backend/dedupe');
const logLimit = getConfig().ssbLogStream?.limit || 1000;
const safeArr = (v) => (Array.isArray(v) ? v : []);
const normalizeTags = (raw) => {
if (raw === undefined || raw === null) return undefined;
if (Array.isArray(raw)) return raw.map((t) => String(t || "").trim()).filter(Boolean);
return String(raw).split(",").map((t) => t.trim()).filter(Boolean);
};
const fileExtra = (extra) => {
if (!extra || typeof extra !== "object") return {};
const out = {};
if (extra.mime) out.mime = String(extra.mime);
if (extra.fileName) out.fileName = String(extra.fileName);
if (typeof extra.torrentKey === "string" && extra.torrentKey.startsWith("%")) out.torrentKey = extra.torrentKey;
if (typeof extra.torrentUrl === "string" && extra.torrentUrl.startsWith("&")) out.torrentUrl = extra.torrentUrl;
if (extra.cipher && typeof extra.cipher === "object" && extra.cipher.key && extra.cipher.manifestBlobId) out.cipher = { v: Number(extra.cipher.v) || 1, key: String(extra.cipher.key), manifestBlobId: String(extra.cipher.manifestBlobId), chunkCount: Number(extra.cipher.chunkCount) || 0 };
return out;
};
const parseBlobId = (blobMarkdown) => {
const s = String(blobMarkdown || "");
const match = s.match(/\((&[^)]+\.sha256)\)/);
if (match) return match[1];
const fallback = s.match(/\(([^)]+)\)/g);
return fallback ? fallback[fallback.length - 1].slice(1, -1) : s || null;
};
const voteSum = (opinions = {}) =>
Object.values(opinions || {}).reduce((s, n) => s + (Number(n) || 0), 0);
module.exports = ({ cooler, tribeCrypto, tribesModel }) => {
let ssb;
const openSsb = async () => {
if (!ssb) ssb = await cooler.open();
return ssb;
};
const tribeHelpers = tribeCrypto ? tribeCrypto.createHelpers(tribesModel) : null;
const encryptIfTribe = tribeHelpers ? tribeHelpers.encryptIfTribe : async (c) => c;
const decryptIfTribe = tribeHelpers ? tribeHelpers.decryptIfTribe : async (c) => c;
const assertReadable = tribeHelpers ? tribeHelpers.assertReadable : () => {};
const decryptIndexNodes = tribeHelpers ? tribeHelpers.decryptIndexNodes : async () => {};
const unwrapForIndex = (msgs) => tribeHelpers ? tribeHelpers.unwrapMessagesForKind(msgs, 'file') : msgs;
const tombFor = async (target, tribeId, author) => tribeHelpers ? tribeHelpers.encryptTombstone(target, tribeId, author) : { type: 'tombstone', target, deletedAt: new Date().toISOString(), author };
const FILE_TYPES = ["file", "fileOpinion", "tombstone"];
const getAllMessages = async (ssbClient) => readTyped(ssbClient, FILE_TYPES, { limit: logLimit, withWindow: true });
const getMsg = async (ssbClient, key) =>
new Promise((resolve) => {
ssbClient.get(key, (err, msg) => (err ? resolve(null) : resolve(msg)));
});
const buildIndex = (messages) => {
const tomb = new Set();
const nodes = new Map();
const authorByKey = new Map();
const tombRequests = [];
const opinionMsgs = [];
for (const m of messages) {
const k = m.key;
const v = m.value || {};
const c = v.content;
if (!c) continue;
if (c.type === "tombstone" && c.target) {
tombRequests.push({ target: c.target, author: v.author });
continue;
}
if (c.type === "fileOpinion") { opinionMsgs.push({ target: c.target, author: v.author, category: c.category }); continue; }
if (c.type !== "file") continue;
const ts = v.timestamp || m.timestamp || 0;
let sizeBytes = 0;
try { sizeBytes = Buffer.byteLength(JSON.stringify(v), "utf8"); } catch (_) { sizeBytes = 0; }
nodes.set(k, { key: k, ts, seq: v.sequence || 0, c, sizeBytes, author: v.author });
authorByKey.set(k, v.author);
}
const vg = versionGroups([...nodes.values()].map(n => ({ key: n.key, author: n.author, seq: n.seq, ts: n.ts, replaces: n.c.replaces || null, claimedAuthor: typeof n.c.author === "string" ? n.c.author : null })));
const rootOf = vg.rootOf;
const contentTipOf = vg.tipOf;
for (const t of tombRequests) {
const targetAuthor = authorByKey.get(t.target);
if (targetAuthor && t.author === targetAuthor) tomb.add(t.target);
}
const opinionsByRoot = new Map();
for (const op of opinionMsgs) { if (!nodes.has(op.target)) continue; const r = rootOf(op.target); if (!opinionsByRoot.has(r)) opinionsByRoot.set(r, []); opinionsByRoot.get(r).push(op); }
const tipByRoot = new Map();
for (const r of vg.roots) tipByRoot.set(r, contentTipOf(r));
const resolveGroup = (root) => {
const contentTip = contentTipOf(root);
const contentNode = nodes.get(contentTip) || nodes.get(root);
const lc = contentNode ? contentNode.c : {};
const opinions = { ...(lc.opinions || {}) };
const voters = safeArr(lc.opinions_inhabitants).slice();
const voterSet = new Set(voters);
for (const op of (opinionsByRoot.get(root) || [])) {
if (voterSet.has(op.author)) continue;
voterSet.add(op.author); voters.push(op.author);
opinions[op.category] = (opinions[op.category] || 0) + 1;
}
return { contentTip, contentNode, opinions, voters };
};
return { tomb, nodes, rootOf, tipByRoot, resolveGroup };
};
const buildFile = (node, rootId, viewerId, agg) => {
const c = node.c || {};
const undec = c.encryptedPayload && c._decrypted === false;
const opinions = agg ? agg.opinions : (c.opinions || {});
const voters = agg ? agg.voters : safeArr(c.opinions_inhabitants);
return {
key: node.key,
rootId,
url: undec ? "" : c.url,
createdAt: c.createdAt || new Date(node.ts).toISOString(),
updatedAt: c.updatedAt || null,
tags: safeArr(c.tags),
author: node.author,
title: undec ? "" : (c.title || ""),
description: undec ? "" : (c.description || ""),
size: c.size || 0,
mime: undec ? "" : (c.mime || ""),
fileName: undec ? "" : (c.fileName || ""),
torrentKey: undec ? "" : (typeof c.torrentKey === "string" && c.torrentKey.startsWith("%") ? c.torrentKey : ""),
torrentUrl: undec ? "" : (typeof c.torrentUrl === "string" && c.torrentUrl.startsWith("&") ? c.torrentUrl : ""),
cipher: undec ? null : (c.cipher && typeof c.cipher === "object" && typeof c.cipher.key === "string" && typeof c.cipher.manifestBlobId === "string" ? { v: Number(c.cipher.v) || 1, key: c.cipher.key, manifestBlobId: c.cipher.manifestBlobId, chunkCount: Number(c.cipher.chunkCount) || 0 } : null),
opinions,
opinions_inhabitants: voters,
hasVoted: viewerId ? voters.includes(viewerId) : false,
tribeId: c.tribeId || null,
encrypted: !!undec,
sizeBytes: node.sizeBytes || 0
};
};
return {
type: "file",
async resolveCurrentId(id) {
const ssbClient = await openSsb();
const messages = await getAllMessages(ssbClient);
const idx = buildIndex(unwrapForIndex(messages));
const tip = idx.tipByRoot.get(idx.rootOf(id)) || id;
if (idx.tomb.has(tip)) throw new Error("File not found");
return tip;
},
async resolveRootId(id) {
const ssbClient = await openSsb();
const messages = await getAllMessages(ssbClient);
const idx = buildIndex(unwrapForIndex(messages));
const root = idx.rootOf(id);
if (idx.tomb.has(idx.tipByRoot.get(root) || id)) throw new Error("File not found");
return root;
},
async createFile(blobMarkdown, tagsRaw, title, description, size, tribeId, extra = null) {
const ssbClient = await openSsb();
const blobId = parseBlobId(blobMarkdown);
const tags = normalizeTags(tagsRaw) || [];
const now = new Date().toISOString();
let content = {
type: "file",
url: blobId,
createdAt: now,
updatedAt: null,
author: ssbClient.id,
tags,
title: title || "",
description: description || "",
size: Number(size) || 0,
opinions: {},
opinions_inhabitants: [],
...(tribeId ? { tribeId } : {}),
...fileExtra(extra)
};
content = await encryptIfTribe(content);
return new Promise((resolve, reject) => {
ssbClient.publish(content, (err, res) => (err ? reject(err) : resolve(res)));
});
},
async createLink(url, tagsRaw, title, description, size, extra = null) {
const ssbClient = await openSsb();
const link = String(url || '').trim();
if (!/^https?:\/\//i.test(link)) throw new Error('Invalid link');
const content = {
type: "file",
url: link,
createdAt: new Date().toISOString(),
updatedAt: null,
author: ssbClient.id,
tags: normalizeTags(tagsRaw) || [],
title: title || "",
description: description || "",
size: Number(size) || 0,
opinions: {},
opinions_inhabitants: [],
...fileExtra(extra)
};
return new Promise((resolve, reject) => {
ssbClient.publish(content, (err, res) => (err ? reject(err) : resolve(res)));
});
},
async updateFileById(id, blobMarkdown, tagsRaw, title, description, extra = null) {
const ssbClient = await openSsb();
const userId = ssbClient.id;
const tipId = await this.resolveCurrentId(id);
const oldMsg = await getMsg(ssbClient, tipId);
if (!oldMsg) throw new Error("File not found");
const oldDec = await decryptIfTribe(oldMsg.content);
if (!oldDec || oldDec.type !== "file") throw new Error("File not found");
assertReadable(oldDec, "File");
if (oldMsg.author !== userId) throw new Error("Not the author");
const aggFile = await this.getFileById(id, userId);
if (aggFile && Object.keys(aggFile.opinions || {}).some(k => (aggFile.opinions[k] || 0) > 0)) throw new Error("Cannot edit file after it has received opinions.");
const tags = tagsRaw !== undefined ? normalizeTags(tagsRaw) || [] : safeArr(oldDec.tags);
const blobId = blobMarkdown ? parseBlobId(blobMarkdown) : null;
const now = new Date().toISOString();
let updated = {
type: "file",
replaces: tipId,
url: blobId || oldDec.url,
tags,
title: title !== undefined ? title || "" : oldDec.title || "",
description: description !== undefined ? description || "" : oldDec.description || "",
size: extra && Number(extra.size) > 0 ? Number(extra.size) : (oldDec.size || 0),
...(extra ? fileExtra(extra) : fileExtra(oldDec)),
opinions: oldDec.opinions || {},
opinions_inhabitants: oldDec.opinions_inhabitants || [],
author: oldDec.author || userId,
...((oldDec.tribeId || oldMsg.content.tribeId) ? { tribeId: oldDec.tribeId || oldMsg.content.tribeId } : {}),
createdAt: oldDec.createdAt,
updatedAt: now
};
updated = await encryptIfTribe(updated);
const result = await new Promise((resolve, reject) => {
ssbClient.publish(updated, (err, res) => (err ? reject(err) : resolve(res)));
});
const tombstone = await tombFor(tipId, oldDec.tribeId || oldMsg.content.tribeId, userId);
await new Promise((res, rej) => ssbClient.publish(tombstone, (e) => (e ? rej(e) : res())));
return result;
},
async deleteFileById(id) {
const ssbClient = await openSsb();
const userId = ssbClient.id;
const tipId = await this.resolveCurrentId(id);
const msg = await getMsg(ssbClient, tipId);
if (!msg) throw new Error("File not found");
const dec = await decryptIfTribe(msg.content);
if (!dec || dec.type !== "file") throw new Error("File not found");
if (msg.author !== userId) throw new Error("Not the author");
const tombstone = await tombFor(tipId, dec.tribeId || msg.content.tribeId, userId);
return new Promise((resolve, reject) => {
ssbClient.publish(tombstone, (err, res) => (err ? reject(err) : resolve(res)));
});
},
async listAll(filterOrOpts = "all", maybeOpts = {}) {
const ssbClient = await openSsb();
const opts = typeof filterOrOpts === "object" ? filterOrOpts : maybeOpts || {};
const filter = (typeof filterOrOpts === "string" ? filterOrOpts : opts.filter || "all") || "all";
const q = String(opts.q || "").trim().toLowerCase();
const sort = String(opts.sort || "recent").trim();
const viewerId = opts.viewerId || ssbClient.id;
const messages = await getAllMessages(ssbClient);
const idx = buildIndex(unwrapForIndex(messages));
await decryptIndexNodes(idx);
const items = [];
for (const [rootId, tipId] of idx.tipByRoot.entries()) {
if (idx.tomb.has(tipId)) continue;
const node = idx.nodes.get(tipId);
if (!node) continue;
items.push(buildFile(node, rootId, viewerId, idx.resolveGroup(rootId)));
}
let list = dedupeBy(items, x => x.url ? [norm(x.author), norm(x.url)].join('|') : null);
const now = Date.now();
if (filter === "mine") list = list.filter((a) => String(a.author) === String(viewerId));
else if (filter === "recent") list = list.filter((a) => new Date(a.createdAt).getTime() >= now - 86400000);
else if (filter === "top") {
list = list.slice().sort((a, b) => voteSum(b.opinions) - voteSum(a.opinions) || new Date(b.createdAt) - new Date(a.createdAt));
}
if (q) {
list = list.filter((a) => {
const title = String(a.title || "").toLowerCase();
const desc = String(a.description || "").toLowerCase();
const tags = safeArr(a.tags).join(" ").toLowerCase();
const author = String(a.author || "").toLowerCase();
return title.includes(q) || desc.includes(q) || tags.includes(q) || author.includes(q);
});
}
if (sort === "top") {
list = list.slice().sort((a, b) => voteSum(b.opinions) - voteSum(a.opinions) || new Date(b.createdAt) - new Date(a.createdAt));
} else if (sort === "oldest") {
list = list.slice().sort((a, b) => new Date(a.createdAt) - new Date(b.createdAt));
} else {
list = list.slice().sort((a, b) => new Date(b.createdAt) - new Date(a.createdAt));
}
return list;
},
async getFileById(id, viewerId = null) {
const ssbClient = await openSsb();
const viewer = viewerId || ssbClient.id;
const messages = await getAllMessages(ssbClient);
const idx = await memoIndex('files', messages, async (m) => { const built = buildIndex(unwrapForIndex(m)); await decryptIndexNodes(built); return built; });
const root = idx.rootOf(id);
if (idx.nodes.has(root) && !idx.tipByRoot.has(root)) throw new Error("File not found");
const agg = idx.resolveGroup(root);
if (agg.contentNode && !idx.tomb.has(agg.contentTip)) return buildFile(agg.contentNode, root, viewer, agg);
const tip = idx.tipByRoot.get(root) || id;
if (idx.tomb.has(tip)) throw new Error("File not found");
const msg = await getMsg(ssbClient, tip);
if (!msg) throw new Error("File not found");
let c = msg.content;
if (tribeCrypto && (c?.encryptedPayload || tribeCrypto.isTribeMsg(c)) && tribesModel) {
const dec = await tribeCrypto.decryptFromTribe(c, tribesModel);
c = dec && !dec._undecryptable ? { ...dec, _decrypted: true } : { ...c, _decrypted: false };
}
if (!c || c.type !== "file") throw new Error("File not found");
return buildFile({ key: tip, ts: msg.timestamp || 0, c, author: msg.author }, root, viewer, agg);
},
async createOpinion(id, category) {
const ssbClient = await openSsb();
const userId = ssbClient.id;
if (!categories.includes(category)) throw new Error("Invalid voting category");
const file = await this.getFileById(id, userId);
if (!file) throw new Error("File not found");
if (safeArr(file.opinions_inhabitants).includes(userId)) throw new Error("Already voted");
const content = { type: "fileOpinion", target: file.rootId, category, createdAt: new Date().toISOString() };
return new Promise((resolve, reject) => {
ssbClient.publish(content, (err, res) => (err ? reject(err) : resolve(res)));
});
}
};
};

42
src/models/fileshare_model.js
View file

@ -5,6 +5,22 @@ const { Readable } = require('stream');
const fsc = require('../backend/fileshare_crypto');
const MAX_CHUNK_SIZE = 5 * 1024 * 1024;
const MAX_PREFETCH_CHUNKS = 64;
const BLOB_REF = /^&[A-Za-z0-9+/]{43}=\.sha256$/;
const DEFAULT_MAX_SHARE_BYTES = 1024 * 1024 * 1024;
const maxShareBytes = () => {
try { return Number(require('../configs/config-manager.js').getConfig().fileShare.maxSize) || DEFAULT_MAX_SHARE_BYTES; } catch (_) { return DEFAULT_MAX_SHARE_BYTES; }
};
const checkManifest = (manifest) => {
const size = Number(manifest && manifest.size);
const chunkSize = Number(manifest && manifest.chunkSize);
const chunks = manifest && manifest.chunks;
if (!Number.isInteger(size) || size < 0 || size > maxShareBytes()) throw new Error('fileshare manifest invalid');
if (!Number.isInteger(chunkSize) || chunkSize <= 0 || chunkSize > MAX_CHUNK_SIZE) throw new Error('fileshare manifest invalid');
if (!Array.isArray(chunks) || chunks.length !== Math.max(1, Math.ceil(size / chunkSize)) || chunks.some(ref => !BLOB_REF.test(String(ref)))) throw new Error('fileshare manifest invalid');
if (manifest.plainSha256 !== null && manifest.plainSha256 !== undefined && !/^[0-9a-f]{64}$/.test(String(manifest.plainSha256))) throw new Error('fileshare manifest invalid');
return manifest;
};
module.exports = ({ cooler }) => {
let ssb;
@ -117,7 +133,7 @@ module.exports = ({ cooler }) => {
const key = fsc.keyFromHex(pointer.key);
await wantBlob(client, pointer.manifestBlobId);
const payload = await getBlob(client, pointer.manifestBlobId);
return fsc.decryptManifest(payload, key);
return checkManifest(fsc.decryptManifest(payload, key));
};
const CHUNK_WANT_TIMEOUT = 30000;
@ -128,11 +144,19 @@ module.exports = ({ cooler }) => {
async function* gen() {
const client = await openSsb();
if (!(await wantBlob(client, pointer.manifestBlobId, CHUNK_WANT_TIMEOUT))) throw new Error('fileshare blob unavailable');
const manifest = fsc.decryptManifest(await getBlob(client, pointer.manifestBlobId), key);
const manifest = checkManifest(fsc.decryptManifest(await getBlob(client, pointer.manifestBlobId), key));
const hash = crypto.createHash('sha256');
let total = 0;
for (const ref of manifest.chunks) {
if (!(await wantBlob(client, ref, CHUNK_WANT_TIMEOUT))) throw new Error('fileshare blob unavailable');
yield fsc.decryptChunk(await getBlob(client, ref), key);
const plain = fsc.decryptChunk(await getBlob(client, ref), key);
total += plain.length;
if (total > manifest.size) throw new Error('fileshare size mismatch');
hash.update(plain);
yield plain;
}
if (total !== manifest.size) throw new Error('fileshare size mismatch');
if (manifest.plainSha256 && hash.digest('hex') !== manifest.plainSha256) throw new Error('fileshare content mismatch');
}
return Readable.from(gen());
};
@ -169,6 +193,16 @@ module.exports = ({ cooler }) => {
}
};
const prefetch = async (pointer) => {
try {
const client = await openSsb();
if (!(await hasBlob(client, pointer.manifestBlobId))) { wantBlob(client, pointer.manifestBlobId, 0); return; }
const manifest = fsc.decryptManifest(await getBlob(client, pointer.manifestBlobId), fsc.keyFromHex(pointer.key));
if (!Array.isArray(manifest.chunks) || manifest.chunks.length > MAX_PREFETCH_CHUNKS || manifest.chunks.some(ref => !BLOB_REF.test(String(ref)))) return;
for (const ref of manifest.chunks) if (!(await hasBlob(client, ref))) wantBlob(client, ref, 0);
} catch (_) {}
};
const removeLocalBlobs = async (pointer) => {
const client = await openSsb();
let removed = 0;
@ -196,5 +230,5 @@ module.exports = ({ cooler }) => {
return pruned;
};
return { createShareFromBuffer, createShareFromFile, openManifest, readShareStream, reassembleToBuffer, isAvailable, ensureAvailable, removeLocalBlobs, pruneExpired };
return { createShareFromBuffer, createShareFromFile, openManifest, readShareStream, reassembleToBuffer, isAvailable, ensureAvailable, prefetch, removeLocalBlobs, pruneExpired };
};

340
src/models/forum_model.js
View file

@ -2,6 +2,8 @@ const pull = require('../server/node_modules/pull-stream');
const crypto = require('crypto');
const { getConfig } = require('../configs/config-manager.js');
const { buildValidatedTombstoneSet } = require('./tombstone_validator');
const { readTyped, requestScope, memoIndex } = require('./typed_log');
const longText = require('../backend/long_text');
const logLimit = getConfig().ssbLogStream?.limit || 1000;
module.exports = ({ cooler, tribeCrypto, forumCrypto }) => {
@ -33,12 +35,13 @@ module.exports = ({ cooler, tribeCrypto, forumCrypto }) => {
const ssbClient = await openSsb();
const ssbKeys = require('../server/node_modules/ssb-keys');
const cfg = require('../server/ssb_config');
const msgs = await new Promise((res, rej) =>
pull(ssbClient.createLogStream({ limit: logLimit }), pull.collect((e, m) => e ? rej(e) : res(m)))
);
const msgs = await readForumLog();
const authorOf = new Map();
for (const m of msgs) if (m && m.key && m.value) authorOf.set(m.key, m.value.author);
for (const m of msgs) {
const c = m.value && m.value.content;
if (!c || c.type !== 'tribe-keys') continue;
if (!c.tribeId || authorOf.get(c.tribeId) !== m.value.author) continue;
const memberKeys = c.memberKeys;
if (!memberKeys || typeof memberKeys !== 'object') continue;
const boxed = memberKeys[ssbClient.id];
@ -60,41 +63,63 @@ module.exports = ({ cooler, tribeCrypto, forumCrypto }) => {
return ssb;
};
const FORUM_TYPES = ['forum', 'forum-invite', 'forum-invite-tombstone', 'forum-open-invite', 'forum-open-invite-tombstone', 'tribe-keys', 'vote', 'tombstone', longText.CHUNK_TYPE];
const chunkIndexOf = (msgs) => longText.indexChunks(
msgs.filter(m => m.value && m.value.content && m.value.content.type === longText.CHUNK_TYPE),
(c) => {
if (!c || !c.encryptedPayload) return c;
const dec = decryptForumContent(c, null);
return dec && !dec._undecryptable ? dec : null;
}
);
const fullText = (idx, c, author) => longText.joinText(c && c.text, c && c.chunks, longText.lookupIn(idx), author);
const readForumLog = async () => {
const ssbClient = await openSsb();
return new Promise((res, rej) => pull(ssbClient.createLogStream({ limit: logLimit }), pull.collect((e, m) => e ? rej(e) : res(m || []))));
return readTyped(ssbClient, FORUM_TYPES, { limit: logLimit, withWindow: true });
};
const scanForumOpenInvite = async (forumId) => {
const messages = await readForumLog();
const authorsByKey = (messages) => {
const out = new Map();
for (const m of messages) if (m && m.key && m.value) out.set(m.key, m.value.author);
return out;
};
const inviteTombstones = (messages, authorOf) => {
const markerTomb = new Set();
const invTomb = new Set();
for (const m of messages) {
const c = m.value && m.value.content;
if (!c) continue;
if (c.type === 'forum-open-invite-tombstone' && typeof c.target === 'string') markerTomb.add(c.target);
if (c.type === 'forum-invite-tombstone' && typeof c.target === 'string') invTomb.add(c.target);
if (!c || typeof c.target !== 'string' || authorOf.get(c.target) !== m.value.author) continue;
if (c.type === 'forum-open-invite-tombstone') markerTomb.add(c.target);
if (c.type === 'forum-invite-tombstone') invTomb.add(c.target);
}
return { markerTomb, invTomb };
};
const scanForumOpenInvite = async (forumId) => {
const messages = await readForumLog();
const authorOf = authorsByKey(messages);
const forumAuthor = authorOf.get(forumId);
if (!forumAuthor) return null;
const { markerTomb, invTomb } = inviteTombstones(messages, authorOf);
let best = null;
for (const m of messages) {
const c = m.value && m.value.content;
if (!c || c.type !== 'forum-open-invite' || c.v !== 1) continue;
if (c.target !== forumId || typeof c.code !== 'string') continue;
if (m.value.author !== forumAuthor) continue;
if (markerTomb.has(m.key)) continue;
if (c.inviteKey && invTomb.has(c.inviteKey)) continue;
const ts = (m.value && m.value.timestamp) || 0;
if (!best || ts > best.ts) best = { code: c.code, by: c.by || m.value.author, markerKey: m.key, inviteKey: c.inviteKey || null, ts };
if (!best || ts > best.ts) best = { code: c.code, by: m.value.author, markerKey: m.key, inviteKey: c.inviteKey || null, ts };
}
return best;
};
async function collectTombstones(ssbClient) {
return new Promise((resolve, reject) => {
pull(
ssbClient.createLogStream({ limit: logLimit }),
pull.collect((err, msgs) => err ? reject(err) : resolve(buildValidatedTombstoneSet(msgs)))
);
});
return deletedOf(await readForumLog());
}
async function findActiveVote(ssbClient, targetId, voter) {
@ -108,16 +133,38 @@ module.exports = ({ cooler, tribeCrypto, forumCrypto }) => {
});
}
async function aggregateVotes(ssbClient, targetId) {
const tombstoned = await collectTombstones(ssbClient);
const deletedOf = (msgs) => memoIndex('forum:deleted', msgs, buildValidatedTombstoneSet);
function aggregateVotes(ssbClient, targetId, known = null) {
const store = requestScope.getStore();
if (!store || store.method !== 'GET') return countVotes(ssbClient, targetId, known);
if (!store.forumVotes) store.forumVotes = new Map();
let pending = store.forumVotes.get(targetId);
if (!pending) { pending = countVotes(ssbClient, targetId, known); store.forumVotes.set(targetId, pending); }
return pending;
}
async function countVotes(ssbClient, targetId, known = null) {
const tombstoned = known || await collectTombstones(ssbClient);
return new Promise((resolve, reject) => {
let positives = 0, negatives = 0;
const latest = new Map();
pull(
ssbClient.links({ source: null, dest: targetId, rel: 'vote', values: true, keys: true }),
pull.filter(link => link.value.content?.vote && !tombstoned.has(link.key)),
pull.drain(
link => link.value.content.vote.value > 0 ? positives++ : negatives++,
err => err ? reject(err) : resolve({ positives, negatives })
link => {
const author = link.value.author;
const ts = Number(link.value.timestamp) || 0;
const seq = Number(link.value.sequence) || 0;
const prev = latest.get(author);
if (!prev || ts > prev.ts || (ts === prev.ts && seq > prev.seq)) latest.set(author, { value: Number(link.value.content.vote.value) || 0, ts, seq });
},
err => {
if (err) return reject(err);
let positives = 0, negatives = 0;
for (const v of latest.values()) { if (v.value > 0) positives++; else if (v.value < 0) negatives++; }
resolve({ positives, negatives });
}
)
);
});
@ -141,16 +188,89 @@ module.exports = ({ cooler, tribeCrypto, forumCrypto }) => {
}
async function getMessageById(id) {
const ssbClient = await openSsb();
const msgs = await new Promise((res, rej) =>
pull(ssbClient.createLogStream({ limit: logLimit }),
pull.collect((err, data) => err ? rej(err) : res(data)))
);
const msgs = await readForumLog();
const msg = msgs.find(m => m.key === id && m.value.content?.type === 'forum');
if (!msg) throw new Error('Message not found');
return { key: msg.key, ...msg.value.content, timestamp: msg.value.timestamp };
return { key: msg.key, ...msg.value.content, author: msg.value.author, timestamp: msg.value.timestamp };
}
const buildForums = async (ssbClient) => {
const msgs = await readForumLog();
const deleted = deletedOf(msgs);
const chunkIdx = chunkIndexOf(msgs);
const decode = (m) => {
const c = m.value && m.value.content;
if (!c) return null;
if (c.encryptedPayload) {
const dec = decryptForumContent(c, m.value.content.root || m.key);
return (dec && !dec._undecryptable) ? dec : null;
}
return c;
};
const forums = msgs
.map(m => ({ m, c: decode(m) }))
.filter(({ m, c }) => c && c.type === 'forum' && !c.root && !deleted.has(m.key))
.map(({ m, c }) => ({ ...c, key: m.key, author: m.value.author, text: fullText(chunkIdx, c, m.value.author) }));
const forumsWithVotes = await Promise.all(
forums.map(async f => {
const { positives, negatives } = await aggregateVotes(ssbClient, f.key, deleted);
return { ...f, positiveVotes: positives, negativeVotes: negatives };
})
);
const repliesByRoot = {};
msgs.forEach(m => {
const cRaw = m.value && m.value.content;
if (!cRaw) return;
const root = cRaw.encryptedPayload ? null : cRaw.root;
if (!root) {
if (!cRaw.encryptedPayload) return;
const decReply = decryptForumContent(cRaw, null);
if (!decReply || decReply._undecryptable || decReply.type !== 'forum' || !decReply.root) return;
if (deleted.has(m.key)) return;
repliesByRoot[decReply.root] = repliesByRoot[decReply.root] || [];
repliesByRoot[decReply.root].push({ key: m.key, text: fullText(chunkIdx, decReply, m.value.author), author: m.value.author, timestamp: m.value.timestamp });
return;
}
if (cRaw.type === 'forum' && root && !deleted.has(m.key)) {
repliesByRoot[root] = repliesByRoot[root] || [];
repliesByRoot[root].push({ key: m.key, text: fullText(chunkIdx, cRaw, m.value.author), author: m.value.author, timestamp: m.value.timestamp });
}
});
const final = await Promise.all(
forumsWithVotes.map(async f => {
const replies = repliesByRoot[f.key] || [];
await Promise.all(replies.map(async (r) => {
const { positives: rp, negatives: rn } = await aggregateVotes(ssbClient, r.key, deleted);
r.positiveVotes = rp;
r.negativeVotes = rn;
r.score = rp - rn;
}));
const replyPos = replies.reduce((sum, r) => sum + (r.positiveVotes || 0), 0);
const replyNeg = replies.reduce((sum, r) => sum + (r.negativeVotes || 0), 0);
const positiveVotes = f.positiveVotes + replyPos;
const negativeVotes = f.negativeVotes + replyNeg;
const score = positiveVotes - negativeVotes;
const participants = new Set(replies.map(r => r.author).concat(f.author));
const messagesCount = replies.length + 1;
const lastMessage =
replies.length
? replies.reduce((a, b) => (new Date(a.timestamp) > new Date(b.timestamp) ? a : b))
: null;
return {
...f,
positiveVotes,
negativeVotes,
score,
participants: Array.from(participants),
messagesCount,
lastMessage,
messages: replies
};
})
);
return final;
};
return {
ingestKeys: async () => { await ingestOwnTribeKeys(); },
@ -168,15 +288,14 @@ module.exports = ({ cooler, tribeCrypto, forumCrypto }) => {
votes_inhabitants: [],
isPrivate: isPrivateFlag
};
let content = plainContent;
let forumKey = null;
if (isPrivateFlag && ownCrypto && tribeCrypto) {
forumKey = ownCrypto.generateTribeKey();
content = tribeCrypto.encryptContent(plainContent, [forumKey], true);
}
const result = await new Promise((resolve, reject) =>
ssbClient.publish(content, (err, res) => err ? reject(err) : resolve(res))
if (isPrivateFlag && ownCrypto && tribeCrypto) forumKey = ownCrypto.generateTribeKey();
const seal = (c) => forumKey ? tribeCrypto.encryptContent(c, [forumKey], true) : c;
const publish = (c) => new Promise((resolve, reject) =>
ssbClient.publish(seal(c), (err, res) => err ? reject(err) : resolve(res))
);
const content = await longText.chunkContent(plainContent, 'text', { maxBytes: forumKey ? longText.CONTENT_BYTES.sealed : longText.CONTENT_BYTES.plain, publish });
const result = await publish(content);
if (forumKey) {
ownCrypto.setKey(result.key, forumKey, 1);
try {
@ -208,14 +327,17 @@ module.exports = ({ cooler, tribeCrypto, forumCrypto }) => {
votes_inhabitants: []
};
if (parentId) content.branch = parentId;
let key = null;
if (isPrivate && ownCrypto && tribeCrypto) {
const key = lookupKey(forumId);
key = lookupKey(forumId);
if (!key) throw new Error('Missing forum key — cannot reply to encrypted forum');
content = tribeCrypto.encryptContent(content, [key], true);
}
return new Promise((resolve, reject) =>
ssbClient.publish(content, (err, res) => err ? reject(err) : resolve(res))
const seal = (c) => key ? tribeCrypto.encryptContent(c, [key], true) : c;
const publish = (c) => new Promise((resolve, reject) =>
ssbClient.publish(seal(c), (err, res) => err ? reject(err) : resolve(res))
);
content = await longText.chunkContent(content, 'text', { maxBytes: key ? longText.CONTENT_BYTES.sealed : longText.CONTENT_BYTES.plain, publish });
return publish(content);
},
generateInvite: async (forumId) => {
@ -226,7 +348,7 @@ module.exports = ({ cooler, tribeCrypto, forumCrypto }) => {
const rawC = rawRoot.content;
const dec = rawC && rawC.encryptedPayload ? decryptForumContent(rawC, forumId) : rawC;
if (dec && dec._undecryptable) throw new Error('Forum is encrypted and cannot be decrypted');
if (dec.author !== userId) throw new Error('Only the author can generate invites');
if (rawRoot.author !== userId) throw new Error('Only the author can generate invites');
if (dec.isPrivate !== true) throw new Error('Only private forums use invitation codes');
const key = lookupKey(forumId);
if (!key) throw new Error('Missing forum key');
@ -252,7 +374,7 @@ module.exports = ({ cooler, tribeCrypto, forumCrypto }) => {
const rawC = rawRoot.content;
const dec = rawC && rawC.encryptedPayload ? decryptForumContent(rawC, forumId) : rawC;
if (dec && dec._undecryptable) throw new Error('Forum is encrypted and cannot be decrypted');
if (dec.author !== userId) throw new Error('Only the author can generate invites');
if (rawRoot.author !== userId) throw new Error('Only the author can generate invites');
if (dec.isPrivate !== true) throw new Error('Only private forums use invitation codes');
if (await scanForumOpenInvite(forumId)) throw new Error('An open invitation already exists');
const key = lookupKey(forumId);
@ -276,9 +398,7 @@ module.exports = ({ cooler, tribeCrypto, forumCrypto }) => {
let author = null;
try {
const rawRoot = await new Promise((res, rej) => ssbClient.get(forumId, (e, m) => e ? rej(e) : res(m)));
const rawC = rawRoot && rawRoot.content;
const dec = rawC && rawC.encryptedPayload ? decryptForumContent(rawC, forumId) : rawC;
author = dec && dec.author;
author = rawRoot && rawRoot.author;
} catch (_) {}
if (rec.by !== userId && author !== userId) throw new Error('Not allowed to remove this invitation');
await new Promise((resolve, reject) => ssbClient.publish({ type: 'forum-open-invite-tombstone', target: rec.markerKey, ts: new Date().toISOString() }, (err) => err ? reject(err) : resolve()));
@ -288,19 +408,15 @@ module.exports = ({ cooler, tribeCrypto, forumCrypto }) => {
joinByInvite: async (code) => {
if (!ownCrypto || !tribeCrypto) throw new Error('Forum crypto unavailable');
const ssbClient = await openSsb();
const messages = await new Promise((res, rej) =>
pull(ssbClient.createLogStream({ limit: logLimit }), pull.collect((e, m) => e ? rej(e) : res(m)))
);
const invTomb = new Set();
for (const m of messages) {
const c = m.value && m.value.content;
if (c && c.type === 'forum-invite-tombstone' && typeof c.target === 'string') invTomb.add(c.target);
}
const messages = await readForumLog();
const authorOf = authorsByKey(messages);
const { invTomb } = inviteTombstones(messages, authorOf);
let matched = null;
for (const m of messages) {
const c = m.value && m.value.content;
if (!c || c.type !== 'forum-invite') continue;
if (invTomb.has(m.key)) continue;
if (!authorOf.get(c.target) || authorOf.get(c.target) !== m.value.author) continue;
try {
const hash = tribeCrypto.hashInviteCode(code, c.salt);
if (hash === c.codeHash) { matched = c; break; }
@ -356,81 +472,10 @@ module.exports = ({ cooler, tribeCrypto, forumCrypto }) => {
listAll: async filter => {
const ssbClient = await openSsb();
const msgs = await new Promise((res, rej) =>
pull(ssbClient.createLogStream({ limit: logLimit }),
pull.collect((err, data) => err ? rej(err) : res(data)))
);
const deleted = buildValidatedTombstoneSet(msgs);
const decode = (m) => {
const c = m.value && m.value.content;
if (!c) return null;
if (c.encryptedPayload) {
const dec = decryptForumContent(c, m.value.content.root || m.key);
return (dec && !dec._undecryptable) ? dec : null;
}
return c;
};
const forums = msgs
.map(m => ({ m, c: decode(m) }))
.filter(({ m, c }) => c && c.type === 'forum' && !c.root && !deleted.has(m.key))
.map(({ m, c }) => ({ ...c, key: m.key }));
const forumsWithVotes = await Promise.all(
forums.map(async f => {
const { positives, negatives } = await aggregateVotes(ssbClient, f.key);
return { ...f, positiveVotes: positives, negativeVotes: negatives };
})
);
const repliesByRoot = {};
msgs.forEach(m => {
const cRaw = m.value && m.value.content;
if (!cRaw) return;
const root = cRaw.encryptedPayload ? null : cRaw.root;
if (!root) {
if (!cRaw.encryptedPayload) return;
const decReply = decryptForumContent(cRaw, null);
if (!decReply || decReply._undecryptable || decReply.type !== 'forum' || !decReply.root) return;
if (deleted.has(m.key)) return;
repliesByRoot[decReply.root] = repliesByRoot[decReply.root] || [];
repliesByRoot[decReply.root].push({ key: m.key, text: decReply.text, author: decReply.author, timestamp: m.value.timestamp });
return;
}
if (cRaw.type === 'forum' && root && !deleted.has(m.key)) {
repliesByRoot[root] = repliesByRoot[root] || [];
repliesByRoot[root].push({ key: m.key, text: cRaw.text, author: cRaw.author, timestamp: m.value.timestamp });
}
});
const final = await Promise.all(
forumsWithVotes.map(async f => {
const replies = repliesByRoot[f.key] || [];
for (let r of replies) {
const { positives: rp, negatives: rn } = await aggregateVotes(ssbClient, r.key);
r.positiveVotes = rp;
r.negativeVotes = rn;
r.score = rp - rn;
}
const replyPos = replies.reduce((sum, r) => sum + (r.positiveVotes || 0), 0);
const replyNeg = replies.reduce((sum, r) => sum + (r.negativeVotes || 0), 0);
const positiveVotes = f.positiveVotes + replyPos;
const negativeVotes = f.negativeVotes + replyNeg;
const score = positiveVotes - negativeVotes;
const participants = new Set(replies.map(r => r.author).concat(f.author));
const messagesCount = replies.length + 1;
const lastMessage =
replies.length
? replies.reduce((a, b) => (new Date(a.timestamp) > new Date(b.timestamp) ? a : b))
: null;
return {
...f,
positiveVotes,
negativeVotes,
score,
participants: Array.from(participants),
messagesCount,
lastMessage,
messages: replies
};
})
);
const userId = ssbClient.id;
const store = requestScope.getStore();
if (store && !store.forumAll) store.forumAll = buildForums(ssbClient);
const final = (await (store ? store.forumAll : buildForums(ssbClient))).slice();
const filtered =
filter === 'mine'
? final.filter(f => f.author === userId)
@ -442,17 +487,14 @@ module.exports = ({ cooler, tribeCrypto, forumCrypto }) => {
getForumById: async id => {
const ssbClient = await openSsb();
const msgs = await new Promise((res, rej) =>
pull(ssbClient.createLogStream({ limit: logLimit }),
pull.collect((err, data) => err ? rej(err) : res(data)))
);
const deleted = buildValidatedTombstoneSet(msgs);
const msgs = await readForumLog();
const deleted = deletedOf(msgs);
const original = msgs.find(m => m.key === id && !deleted.has(m.key));
if (!original || original.value.content?.type !== 'forum') throw new Error('Forum not found');
const rawBase = original.value.content;
const base = rawBase.encryptedPayload ? decryptForumContent(rawBase, id) : rawBase;
if (base && base._undecryptable) throw new Error('Forum is encrypted and cannot be decrypted with available keys');
const { positives, negatives } = await aggregateVotes(ssbClient, id);
const { positives, negatives } = await aggregateVotes(ssbClient, id, deleted);
const replyAuthors = [];
for (const m of msgs) {
if (deleted.has(m.key)) continue;
@ -464,11 +506,13 @@ module.exports = ({ cooler, tribeCrypto, forumCrypto }) => {
if (!dec || dec._undecryptable) continue;
rc = dec;
}
if (rc.type === 'forum' && rc.root === id) replyAuthors.push(rc.author);
if (rc.type === 'forum' && rc.root === id) replyAuthors.push(m.value.author);
}
const participants = Array.from(new Set(replyAuthors.concat(base.author).filter(Boolean)));
const participants = Array.from(new Set(replyAuthors.concat(original.value.author).filter(Boolean)));
return {
...base,
author: original.value.author,
text: fullText(chunkIndexOf(msgs), base, original.value.author),
key: id,
positiveVotes: positives,
negativeVotes: negatives,
@ -479,11 +523,9 @@ module.exports = ({ cooler, tribeCrypto, forumCrypto }) => {
getMessagesByForumId: async forumId => {
const ssbClient = await openSsb();
const msgs = await new Promise((res, rej) =>
pull(ssbClient.createLogStream({ limit: logLimit }),
pull.collect((err, data) => err ? rej(err) : res(data)))
);
const deleted = buildValidatedTombstoneSet(msgs);
const msgs = await readForumLog();
const deleted = deletedOf(msgs);
const chunkIdx = chunkIndexOf(msgs);
const decodeReply = (m) => {
const c = m.value && m.value.content;
if (!c || c.type !== 'forum') return null;
@ -500,18 +542,20 @@ module.exports = ({ cooler, tribeCrypto, forumCrypto }) => {
.filter(r => r && !deleted.has(r.m.key))
.map(({ c, m }) => ({
key: m.key,
text: c.text,
author: c.author,
text: fullText(chunkIdx, c, m.value.author),
author: m.value.author,
timestamp: m.value.timestamp,
parent: c.branch || null
}));
for (let r of replies) {
const { positives: rp, negatives: rn } = await aggregateVotes(ssbClient, r.key);
r.positiveVotes = rp;
r.negativeVotes = rn;
r.score = rp - rn;
}
const { positives: p, negatives: n } = await aggregateVotes(ssbClient, forumId);
const [, { positives: p, negatives: n }] = await Promise.all([
Promise.all(replies.map(async (r) => {
const { positives: rp, negatives: rn } = await aggregateVotes(ssbClient, r.key, deleted);
r.positiveVotes = rp;
r.negativeVotes = rn;
r.score = rp - rn;
})),
aggregateVotes(ssbClient, forumId, deleted)
]);
const replyPos = replies.reduce((sum, r) => sum + (r.positiveVotes || 0), 0);
const replyNeg = replies.reduce((sum, r) => sum + (r.negativeVotes || 0), 0);
const positiveVotes = p + replyPos;

12
src/models/games_model.js
View file

@ -1,11 +1,14 @@
const pull = require('../server/node_modules/pull-stream');
const { getConfig } = require('../configs/config-manager.js');
const { readTyped } = require('./typed_log');
const logLimit = getConfig().ssbLogStream?.limit || 5000;
const GAME_TYPES = ['gameScore'];
const VALID_GAMES = new Set([
'cocoland', 'ecoinflow', 'neoninfiltrator', 'spaceinvaders', 'arkanoid', 'pingpong',
'asteroids', 'tiktaktoe', 'flipflop',
'8ball', 'artillery', 'labyrinth', 'cocoman', 'tetris'
'8ball', 'artillery', 'labyrinth', 'cocoman', 'tetris', 'minesweeper'
]);
module.exports = ({ cooler }) => {
@ -16,12 +19,7 @@ module.exports = ({ cooler }) => {
};
async function readAll(ssbClient) {
return new Promise((resolve, reject) => {
pull(
ssbClient.createLogStream({ limit: logLimit }),
pull.collect((err, results) => (err ? reject(err) : resolve(results)))
);
});
return readTyped(ssbClient, GAME_TYPES, { limit: logLimit });
}
return {

95
src/models/housing_model.js
View file

@ -2,7 +2,9 @@ const pull = require("../server/node_modules/pull-stream")
const moment = require("../server/node_modules/moment")
const categories = require("../backend/opinion_categories")
const { getConfig } = require("../configs/config-manager.js")
const { dedupeByPreferring } = require('../backend/dedupe')
const { dedupeByPreferring, versionGroups } = require('../backend/dedupe')
const { buildValidatedTombstoneSet } = require('./tombstone_validator')
const { readTyped, memoIndex } = require('./typed_log')
const logLimit = getConfig().ssbLogStream?.limit || 1000
const HOUSING_TYPES = ["sale", "rent", "couchsurfing"]
@ -111,21 +113,14 @@ module.exports = ({ cooler, tribeCrypto }) => {
})
})
const readAll = async (ssbClient) =>
new Promise((resolve, reject) =>
pull(
ssbClient.createLogStream({ limit: logLimit }),
pull.collect((err, msgs) => err ? reject(err) : resolve(msgs))
)
)
const HOUSING_MSG_TYPES = ["housing", "housingOpinion", "housingRequest", "tombstone"]
const readAll = async (ssbClient) => readTyped(ssbClient, HOUSING_MSG_TYPES, { limit: logLimit, withWindow: true, withPrivate: true })
const buildIndex = (messages, ssbClient) => {
const tomb = new Set()
const tomb = buildValidatedTombstoneSet(messages)
const nodes = new Map()
const parent = new Map()
const child = new Map()
const naiveReplaces = new Map()
const requestLatest = new Map()
const requestMsgs = []
const opinionMsgs = []
for (const m of messages) {
@ -134,11 +129,8 @@ module.exports = ({ cooler, tribeCrypto }) => {
const c = v.content
if (!c || typeof c !== "object") continue
if (c.type === "tombstone" && c.target) { tomb.add(c.target); continue }
if (c.type === "housing") {
nodes.set(key, { key, ts: v.timestamp || m.timestamp || 0, c, author: v.author })
if (c.replaces) naiveReplaces.set(key, c.replaces)
nodes.set(key, { key, ts: v.timestamp || m.timestamp || 0, seq: v.sequence || 0, c, author: v.author })
continue
}
@ -150,42 +142,11 @@ module.exports = ({ cooler, tribeCrypto }) => {
if (c.type === "housingRequest" && c.housingId) {
const author = v.author
if (!author) continue
const ts = v.timestamp || m.timestamp || 0
const k = `${c.housingId}::${author}`
const prev = requestLatest.get(k)
if (!prev || ts >= prev.ts) requestLatest.set(k, { ts, value: !!c.value, author, housingId: c.housingId })
requestMsgs.push({ ts: v.timestamp || m.timestamp || 0, value: !!c.value, author, housingId: c.housingId })
continue
}
}
for (const [key, replacesId] of naiveReplaces.entries()) {
const node = nodes.get(key)
if (!node) continue
const orig = nodes.get(replacesId)
if (!orig) continue
if (String(orig.author) !== String(node.author)) { nodes.delete(key); continue }
parent.set(key, replacesId)
child.set(replacesId, key)
}
const rootOf = (id) => {
let cur = id, guard = 0
while (parent.has(cur) && guard++ < 100000) cur = parent.get(cur)
return cur
}
const tipOf = (id) => {
let cur = id, guard = 0
while (child.has(cur) && guard++ < 100000) cur = child.get(cur)
return cur
}
const roots = new Set()
for (const id of nodes.keys()) roots.add(rootOf(id))
const tipByRoot = new Map()
for (const r of roots) tipByRoot.set(r, tipOf(r))
if (ssbClient) {
for (const m of messages) {
if (typeof m.value?.content !== 'string') continue
@ -195,14 +156,26 @@ module.exports = ({ cooler, tribeCrypto }) => {
if (!c || c.type !== 'housingRequest' || !c.housingId) continue
const author = dec.value.author
if (!author) continue
const ts = dec.value.timestamp || m.timestamp || 0
const k = `${c.housingId}::${author}`
const prev = requestLatest.get(k)
if (!prev || ts >= prev.ts) requestLatest.set(k, { ts, value: !!c.value, author, housingId: c.housingId })
requestMsgs.push({ ts: dec.value.timestamp || m.timestamp || 0, value: !!c.value, author, housingId: c.housingId })
} catch {}
}
}
const vg = versionGroups(Array.from(nodes.values()).map(n => ({ key: n.key, author: n.author, seq: n.seq, ts: n.ts, replaces: typeof n.c.replaces === "string" ? n.c.replaces : null, claimedAuthor: typeof n.c.author === "string" ? n.c.author : null })))
const rootOf = vg.rootOf
const tipOf = vg.tipOf
const tipByRoot = new Map()
for (const r of vg.roots) tipByRoot.set(r, tipOf(r))
const requestLatest = new Map()
for (const req of requestMsgs) {
const housingId = rootOf(req.housingId)
const k = `${housingId}::${req.author}`
const prev = requestLatest.get(k)
if (!prev || req.ts >= prev.ts) requestLatest.set(k, { ...req, housingId })
}
const requestsByRoot = new Map()
const everRequestedByRoot = new Map()
for (const { housingId, author, value } of requestLatest.values()) {
@ -236,7 +209,7 @@ module.exports = ({ cooler, tribeCrypto }) => {
return { opinions, opinions_inhabitants: voters }
}
return { tomb, nodes, parent, child, rootOf, tipOf, tipByRoot, requestsByRoot, everRequestedByRoot, aggregateFor }
return { tomb, nodes, vg, rootOf, tipOf, tipByRoot, requestsByRoot, everRequestedByRoot, aggregateFor }
}
const buildObject = (node, rootId, idx, viewer) => {
@ -355,7 +328,7 @@ module.exports = ({ cooler, tribeCrypto }) => {
const ssbClient = await openSsb()
const idx = buildIndex(await readAll(ssbClient), ssbClient)
const tip = idx.tipOf(id)
if (idx.tomb.has(tip)) throw new Error("Housing not found")
if (idx.tomb.has(tip) || idx.vg.isForeignCopy(idx.rootOf(tip))) throw new Error("Housing not found")
return tip
},
@ -363,7 +336,7 @@ module.exports = ({ cooler, tribeCrypto }) => {
const ssbClient = await openSsb()
const idx = buildIndex(await readAll(ssbClient), ssbClient)
const tip = idx.tipOf(id)
if (idx.tomb.has(tip)) throw new Error("Housing not found")
if (idx.tomb.has(tip) || idx.vg.isForeignCopy(idx.rootOf(tip))) throw new Error("Housing not found")
return idx.rootOf(tip)
},
@ -371,7 +344,7 @@ module.exports = ({ cooler, tribeCrypto }) => {
const ssbClient = await openSsb()
const idx = buildIndex(await readAll(ssbClient), ssbClient)
const tipId = idx.tipOf(id)
if (idx.tomb.has(tipId)) throw new Error("Housing not found")
if (idx.tomb.has(tipId) || idx.vg.isForeignCopy(idx.rootOf(tipId))) throw new Error("Housing not found")
const node = idx.nodes.get(tipId)
if (!node || !node.c) throw new Error("Housing not found")
if ((node.author || node.c.author) !== ssbClient.id) throw new Error("Unauthorized")
@ -433,7 +406,7 @@ module.exports = ({ cooler, tribeCrypto }) => {
const ssbClient = await openSsb()
const idx = buildIndex(await readAll(ssbClient), ssbClient)
const tipId = idx.tipOf(id)
if (idx.tomb.has(tipId)) throw new Error("Housing not found")
if (idx.tomb.has(tipId) || idx.vg.isForeignCopy(idx.rootOf(tipId))) throw new Error("Housing not found")
const node = idx.nodes.get(tipId)
if (!node) throw new Error("Housing not found")
if ((node.author || node.c.author) !== ssbClient.id) throw new Error("Unauthorized")
@ -468,7 +441,7 @@ module.exports = ({ cooler, tribeCrypto }) => {
const me = ssbClient.id
const idx = buildIndex(await readAll(ssbClient), ssbClient)
const tipId = idx.tipOf(id)
if (idx.tomb.has(tipId)) throw new Error("Housing not found")
if (idx.tomb.has(tipId) || idx.vg.isForeignCopy(idx.rootOf(tipId))) throw new Error("Housing not found")
const node = idx.nodes.get(tipId)
if (!node) throw new Error("Housing not found")
const rootId = idx.rootOf(tipId)
@ -539,9 +512,9 @@ module.exports = ({ cooler, tribeCrypto }) => {
async getHousingById(id, viewerId = null) {
const ssbClient = await openSsb()
const viewer = viewerId || ssbClient.id
const idx = buildIndex(await readAll(ssbClient), ssbClient)
const idx = memoIndex('housing', await readAll(ssbClient), (m) => buildIndex(m, ssbClient))
const tipId = idx.tipOf(id)
if (idx.tomb.has(tipId)) throw new Error("Housing not found")
if (idx.tomb.has(tipId) || idx.vg.isForeignCopy(idx.rootOf(tipId))) throw new Error("Housing not found")
const node = idx.nodes.get(tipId)
if (!node) throw new Error("Housing not found")
const rootId = idx.rootOf(tipId)

83
src/models/images_model.js
View file

@ -2,7 +2,8 @@ const pull = require("../server/node_modules/pull-stream");
const { getConfig } = require("../configs/config-manager.js");
const categories = require("../backend/opinion_categories");
const { buildValidatedTombstoneSet } = require('./tombstone_validator');
const { dedupeBy, norm } = require('../backend/dedupe');
const { readTyped, memoIndex } = require('./typed_log');
const { dedupeBy, norm, versionGroups } = require('../backend/dedupe');
const logLimit = getConfig().ssbLogStream?.limit || 1000;
@ -31,13 +32,9 @@ module.exports = ({ cooler }) => {
return ssb;
};
const getAllMessages = async (ssbClient) =>
new Promise((resolve, reject) => {
pull(
ssbClient.createLogStream({ limit: logLimit }),
pull.collect((err, msgs) => (err ? reject(err) : resolve(msgs)))
);
});
const IMAGE_TYPES = ["image", "imageOpinion", "tombstone"];
const getAllMessages = async (ssbClient) => readTyped(ssbClient, IMAGE_TYPES, { limit: logLimit });
const getMsg = async (ssbClient, key) =>
new Promise((resolve, reject) => {
@ -47,9 +44,6 @@ module.exports = ({ cooler }) => {
const buildIndex = (messages) => {
const tomb = buildValidatedTombstoneSet(messages);
const nodes = new Map();
const parent = new Map();
const child = new Map();
const strictChild = new Map();
const opinionMsgs = [];
for (const m of messages) {
@ -65,48 +59,16 @@ module.exports = ({ cooler }) => {
const ts = v.timestamp || m.timestamp || 0;
let sizeBytes = 0;
try { sizeBytes = Buffer.byteLength(JSON.stringify(v), "utf8"); } catch (_) { sizeBytes = 0; }
nodes.set(k, { key: k, ts, c, sizeBytes, author: v.author });
if (c.replaces) {
parent.set(k, c.replaces);
child.set(c.replaces, k);
}
nodes.set(k, { key: k, ts, seq: v.sequence || 0, c, sizeBytes, author: v.author });
}
for (const [k, t] of Array.from(parent.entries())) {
const cn = nodes.get(k);
const pn = nodes.get(t);
if (!pn) { parent.delete(k); if (child.get(t) === k) child.delete(t); continue; }
if (!cn || String(cn.author) !== String(pn.author)) { parent.delete(k); if (child.get(t) === k) child.delete(t); nodes.delete(k); }
}
for (const [k, node] of nodes) {
const t = node.c.replaces;
if (t) { const orig = nodes.get(t); if (orig && orig.author === node.author) strictChild.set(t, k); }
}
const rootOf = (id) => {
let cur = id, g = 0;
while (parent.has(cur) && nodes.has(parent.get(cur)) && g++ < 100000) cur = parent.get(cur);
return cur;
};
const contentTipOf = (root) => {
let cur = root, g = 0;
while (strictChild.has(cur) && g++ < 100000) cur = strictChild.get(cur);
const n = nodes.get(cur), rn = nodes.get(root);
return (n && rn && n.author === rn.author) ? cur : root;
};
const roots = new Set();
for (const id of nodes.keys()) roots.add(rootOf(id));
const vg = versionGroups([...nodes.values()].map(n => ({ key: n.key, author: n.author, seq: n.seq, ts: n.ts, replaces: n.c.replaces || null, claimedAuthor: typeof n.c.author === "string" ? n.c.author : null })));
const rootOf = vg.rootOf;
const contentTipOf = vg.tipOf;
const opinionsByRoot = new Map();
for (const op of opinionMsgs) { if (!nodes.has(op.target)) continue; const r = rootOf(op.target); if (!opinionsByRoot.has(r)) opinionsByRoot.set(r, []); opinionsByRoot.get(r).push(op); }
const forward = new Map();
for (const [newId, oldId] of parent.entries()) forward.set(oldId, newId);
const resolveGroup = (root) => {
const contentTip = contentTipOf(root);
const contentNode = nodes.get(contentTip) || nodes.get(root);
@ -123,9 +85,9 @@ module.exports = ({ cooler }) => {
};
const tipByRoot = new Map();
for (const r of roots) tipByRoot.set(r, contentTipOf(r));
for (const r of vg.roots) tipByRoot.set(r, contentTipOf(r));
return { tomb, nodes, parent, child, rootOf, tipByRoot, forward, resolveGroup };
return { tomb, nodes, rootOf, tipByRoot, resolveGroup };
};
const buildImage = (node, rootId, viewerId, agg) => {
@ -141,6 +103,7 @@ module.exports = ({ cooler }) => {
author: node.author,
title: c.title || "",
description: c.description || "",
license: c.license || "",
mapUrl: c.mapUrl || "",
opinions: agg ? agg.opinions : (c.opinions || {}),
opinions_inhabitants: voters,
@ -167,16 +130,12 @@ module.exports = ({ cooler }) => {
const messages = await getAllMessages(ssbClient);
const idx = buildIndex(messages);
let tip = id;
while (idx.forward.has(tip)) tip = idx.forward.get(tip);
if (idx.tomb.has(tip)) throw new Error("Image not found");
let root = tip;
while (idx.parent.has(root)) root = idx.parent.get(root);
const root = idx.rootOf(id);
if (idx.tomb.has(idx.tipByRoot.get(root) || id)) throw new Error("Image not found");
return root;
},
async createImage(blobMarkdown, tagsRaw, title, description, mapUrl) {
async createImage(blobMarkdown, tagsRaw, title, description, mapUrl, license) {
const ssbClient = await openSsb();
const blobId = parseBlobId(blobMarkdown);
const tags = normalizeTags(tagsRaw) || [];
@ -193,7 +152,8 @@ module.exports = ({ cooler }) => {
description: description || "",
mapUrl: mapUrl || "",
opinions: {},
opinions_inhabitants: []
opinions_inhabitants: [],
...(license ? { license } : {})
};
return new Promise((resolve, reject) => {
@ -201,7 +161,7 @@ module.exports = ({ cooler }) => {
});
},
async updateImageById(id, blobMarkdown, tagsRaw, title, description, mapUrl) {
async updateImageById(id, blobMarkdown, tagsRaw, title, description, mapUrl, license) {
const ssbClient = await openSsb();
const userId = ssbClient.id;
const tipId = await this.resolveCurrentId(id);
@ -225,7 +185,8 @@ module.exports = ({ cooler }) => {
description: description !== undefined ? description || "" : oldMsg.content.description || "",
mapUrl: mapUrl !== undefined ? mapUrl || "" : oldMsg.content.mapUrl || "",
createdAt: oldMsg.content.createdAt,
updatedAt: now
updatedAt: now,
...(license !== undefined ? { license } : {})
};
const tombstone = { type: "tombstone", target: tipId, deletedAt: now, author: userId };
@ -311,10 +272,10 @@ module.exports = ({ cooler }) => {
const viewer = viewerId || ssbClient.id;
const messages = await getAllMessages(ssbClient);
const idx = buildIndex(messages);
const idx = memoIndex('images', messages, buildIndex);
const rootId = idx.rootOf(id);
const agg = idx.resolveGroup(rootId);
if (!agg.contentNode || idx.tomb.has(agg.contentTip)) throw new Error("Image not found");
if (!idx.tipByRoot.has(rootId) || !agg.contentNode || idx.tomb.has(agg.contentTip)) throw new Error("Image not found");
return buildImage(agg.contentNode, rootId, viewer, agg);
},

11
src/models/industry_model.js
View file

@ -1,6 +1,7 @@
const pull = require("../server/node_modules/pull-stream")
const { getConfig } = require("../configs/config-manager.js")
const { buildValidatedTombstoneSet } = require('./tombstone_validator')
const { readTyped } = require('./typed_log')
const logLimit = (getConfig().ssbLogStream && getConfig().ssbLogStream.limit) || 1000
module.exports = ({ cooler }) => {
@ -42,13 +43,10 @@ module.exports = ({ cooler }) => {
return Number.isFinite(x) && x >= 0 ? x : 0
}
const INDUSTRY_TYPES = [TYPE, "industryMember", "industryInvite", "industryApply", "industryVote", "industryOpinion", "industryBlueprint", "industryBuild", "industryBuildStatus", "industryBuildDates", "industryBuildUpdate", "industryContribution", "industryAllocation", "tombstone"]
async function getAllMsgs(ssbClient) {
return new Promise((r, j) => {
pull(
ssbClient.createLogStream({ limit: logLimit }),
pull.collect((e, m) => (e ? j(e) : r(m)))
)
})
return readTyped(ssbClient, INDUSTRY_TYPES, { limit: logLimit })
}
function chainMaps(nodeMap) {
@ -566,6 +564,7 @@ module.exports = ({ cooler }) => {
}
const f = String(filter || "ALL").toUpperCase()
if (f === "MINE") list = list.filter(x => x.steward === uid)
else if (f === "RECENT") list = list.filter(x => (Date.parse(x.createdAt || "") || 0) >= Date.now() - 86400000)
else if (f === "MEMBER") list = list.filter(x => Array.isArray(x.members) && x.members.includes(uid))
else if (f === "ACTIVE") list = list.filter(x => x.status === "ACTIVE")
else if (f === "PAUSED") list = list.filter(x => x.status === "PAUSED")

220
src/models/inhabitants_model.js
View file

@ -1,11 +1,7 @@
const pull = require('../server/node_modules/pull-stream');
const ssbClientGUI = require("../client/gui");
const coolerInstance = ssbClientGUI({ offline: require('../server/ssb_config').offline });
const { readTyped, CONTENT_TYPES, requestScope, authorActivity } = require('./typed_log');
const { isContentVisibleTo } = require('./content_visibility');
const models = require("../models/main_models");
const { about, friend } = models({
cooler: coolerInstance,
isPublic: require('../server/ssb_config').public,
});
const { getConfig } = require('../configs/config-manager.js');
const logLimit = getConfig().ssbLogStream?.limit || 1000;
@ -15,36 +11,48 @@ function toImageUrl(imgId, size=256){
return `/image/${size}/${encodeURIComponent(imgId)}`;
}
module.exports = ({ cooler }) => {
const MIN_SUGGESTION_AFFINITY = 0.1;
const MAX_SUGGESTED = 12;
module.exports = ({ cooler, tribesModel = null, dataModel = null }) => {
const { about, friend } = models({ cooler, isPublic: require('../server/ssb_config').public });
let ssb;
const openSsb = async () => { if (!ssb) ssb = await cooler.open(); return ssb; };
async function getLastKarmaScore(feedId) {
const bytesPerAuthor = (ssbClient) => readTyped(ssbClient, CONTENT_TYPES, { limit: logLimit, withWindow: true }).then((msgs) => {
const acc = {};
for (const m of msgs) {
const author = m && m.value && m.value.author;
if (!author) continue;
try { acc[author] = (acc[author] || 0) + Buffer.byteLength(JSON.stringify(m.value), 'utf8'); } catch (_) {}
}
return acc;
}).catch(() => ({}));
async function latestKarmaByAuthor() {
const ssbClient = await openSsb();
return new Promise(resolve => {
const out = new Map();
const src = ssbClient.messagesByType
? ssbClient.messagesByType({ type: "karmaScore", reverse: true })
: ssbClient.createLogStream && ssbClient.createLogStream({ reverse: true });
if (!src) return resolve(0);
if (!src) return resolve(out);
pull(
src,
pull.filter(msg => {
pull.drain(msg => {
const v = msg.value || msg;
const c = v.content || {};
return v.author === feedId && c.type === "karmaScore" && typeof c.karmaScore !== "undefined";
}),
pull.take(1),
pull.collect((err, arr) => {
if (err || !arr || !arr.length) return resolve(0);
const v = arr[0].value || arr[0];
resolve(v.content.karmaScore || 0);
})
if (c.type !== "karmaScore" || typeof c.karmaScore === "undefined" || out.has(v.author)) return;
out.set(v.author, c.karmaScore || 0);
}, () => resolve(out))
);
});
}
async function getLastActivityTimestamp(feedId) {
const ssbClient = await openSsb();
const activity = await authorActivity(ssbClient);
if (activity) return activity.get(feedId) || null;
const norm = (t) => (t && t < 1e12 ? t * 1000 : t || 0);
return new Promise((resolve) => {
pull(
@ -82,21 +90,34 @@ module.exports = ({ cooler }) => {
}
};
async function listAllBase(ssbClient) {
const authorsMsgs = await new Promise((res, rej) => {
pull(
ssbClient.createLogStream({ limit: logLimit, reverse: true }),
pull.filter(msg => !!msg.value?.author && msg.value?.content?.type !== 'tombstone'),
pull.collect((err, msgs) => err ? rej(err) : res(msgs))
);
});
const uniqueFeedIds = Array.from(new Set(authorsMsgs.map(r => r.value.author).filter(Boolean)));
const publicMode = require('../server/ssb_config').public;
const imageUrlFrom = (raw, size = 256) => {
const img = raw == null || raw.link == null ? `&${"0".repeat(43)}=.sha256` : (typeof raw.link === 'string' ? raw.link : raw);
const id = typeof img === 'string' ? img : (img && (img.link || img.url));
return toImageUrl(id, size);
};
async function listAllBase(ssbClient, onlyIds = null) {
const store = requestScope.getStore();
const memoKey = onlyIds ? null : 'inhabitantsBase';
if (memoKey && store && store[memoKey]) return store[memoKey];
const pending = buildBase(ssbClient, onlyIds);
if (memoKey && store) store[memoKey] = pending;
return pending;
}
async function buildBase(ssbClient, onlyIds) {
const only = onlyIds ? new Set(onlyIds) : null;
const authorsMsgs = (await readTyped(ssbClient, CONTENT_TYPES, { limit: logLimit, withWindow: true })).filter(msg => !!msg.value?.author && msg.value?.content?.type !== 'tombstone').reverse();
const uniqueFeedIds = Array.from(new Set(authorsMsgs.map(r => r.value.author).filter(Boolean))).filter(id => !only || only.has(id));
const profiles = publicMode ? null : await about.profiles().catch(() => null);
const users = await Promise.all(
uniqueFeedIds.map(async (feedId) => {
const rawName = await about.name(feedId);
const p = profiles ? (profiles.get(feedId) || {}) : null;
const rawName = p ? (p.name || feedId.slice(1, 9)) : await about.name(feedId);
const name = rawName || feedId.slice(0, 10);
const description = await about.description(feedId);
const photo = await fetchUserImageUrl(feedId, 256);
const description = p ? (p.description || "") : await about.description(feedId);
const photo = p ? imageUrlFrom(p.image, 256) : await fetchUserImageUrl(feedId, 256);
const lastActivityTs = await getLastActivityTimestamp(feedId);
const { bucket, range } = bucketLastActivity(lastActivityTs);
return { id: feedId, name, description, photo, lastActivityTs, lastActivityBucket: bucket, lastActivityRange: range };
@ -115,8 +136,14 @@ module.exports = ({ cooler }) => {
}
return {
async countInhabitants() {
const ssbClient = await openSsb();
const msgs = await readTyped(ssbClient, CONTENT_TYPES, { limit: logLimit, withWindow: true });
return new Set(msgs.filter(msg => !!msg.value?.author && msg.value?.content?.type !== 'tombstone').map(msg => msg.value.author)).size;
},
async listInhabitants(options = {}) {
const { filter = 'all', search = '', location = '', language = '', skills = '', includeInactive = false } = options;
const { filter = 'all', search = '', location = '', language = '', skills = '', includeInactive = false, ids = null } = options;
const ssbClient = await openSsb();
const userId = ssbClient.id;
@ -131,7 +158,7 @@ module.exports = ({ cooler }) => {
}
if (filter === 'all' || filter === 'TOP KARMA' || filter === 'TOP ACTIVITY' || filter === 'TOP INACTIVITY' || filter === 'TOP ECO') {
let users = await listAllBase(ssbClient);
let users = (await listAllBase(ssbClient, ids)).slice();
if (filter !== 'TOP ACTIVITY' && filter !== 'TOP INACTIVITY') {
users = filterInactive(users);
}
@ -143,23 +170,12 @@ module.exports = ({ cooler }) => {
(u.id || '').toLowerCase().includes(q)
);
}
const bytesByAuthor = await new Promise((res) => {
pull(
ssbClient.createLogStream({ limit: logLimit }),
pull.collect((err, msgs) => {
if (err || !Array.isArray(msgs)) return res({});
const acc = {};
for (const m of msgs) {
const author = m && m.value && m.value.author;
if (!author) continue;
try { acc[author] = (acc[author] || 0) + Buffer.byteLength(JSON.stringify(m.value), 'utf8'); } catch (_) {}
}
res(acc);
})
);
});
const store = requestScope.getStore();
if (store && !store.inhabitantBytes) store.inhabitantBytes = bytesPerAuthor(ssbClient);
const bytesByAuthor = await (store ? store.inhabitantBytes : bytesPerAuthor(ssbClient));
const karmaOf = await latestKarmaByAuthor();
const withMetrics = await Promise.all(users.map(async u => {
const karmaScore = await getLastKarmaScore(u.id);
const karmaScore = karmaOf.get(u.id) || 0;
const bytes = (bytesByAuthor && bytesByAuthor[u.id]) || 0;
const carbonGrams = (bytes / (1024 * 1024)) * 0.095;
if (filter === 'TOP ECO') {
@ -199,80 +215,54 @@ module.exports = ({ cooler }) => {
if (filter === 'SUGGESTED') {
const base = await listAllBase(ssbClient);
const active = filterInactive(base);
const cvRecords = await new Promise((res) => {
pull(
ssbClient.createLogStream({ limit: logLimit, reverse: true }),
pull.filter(msg => msg && msg.value && msg.value.content && msg.value.content.type === 'curriculum'),
pull.collect((err, msgs) => err ? res([]) : res(msgs))
);
});
const cvByAuthor = new Map();
for (const r of cvRecords) {
const c = r.value && r.value.content;
if (c && c.author && !cvByAuthor.has(c.author)) cvByAuthor.set(c.author, c);
}
const extractSkills = (cv) => cv ? [
...(cv.personalSkills || []),
...(cv.oasisSkills || []),
...(cv.educationalSkills || []),
...(cv.professionalSkills || [])
].map(s => String(s || '').toLowerCase()).filter(Boolean) : [];
const mecv = await this.getCVByUserId().catch(() => null);
const mySkills = extractSkills(mecv);
const affinities = dataModel ? await dataModel.authorAffinities().catch(() => null) : null;
const byAuthor = affinities ? affinities.byAuthor : new Map();
const karmaOf = await latestKarmaByAuthor();
const rels = await Promise.all(
active.map(async u => {
if (u.id === userId) return null;
const rel = await friend.getRelationship(u.id).catch(() => ({}));
const n = normalizeRel(rel);
if (n.iFollow || n.blocking || n.blockedBy) return null;
const karmaScore = await getLastKarmaScore(u.id);
const theirSkills = extractSkills(cvByAuthor.get(u.id));
const commonSkills = mySkills.length && theirSkills.length
? Array.from(new Set(mySkills.filter(s => theirSkills.includes(s))))
: [];
const followsMeBonus = n.followsMe ? 20 : 0;
const karmaBonus = Math.min(20, Math.log10(1 + Math.max(0, karmaScore)) * 5);
const skillBonus = commonSkills.length * 4;
const activityBonus = u.lastActivityBucket === 'green' ? 5 : (u.lastActivityBucket === 'orange' ? 2 : 0);
const suggestionScore = followsMeBonus + karmaBonus + skillBonus + activityBonus;
return { user: u, rel: n, karmaScore, commonSkills, suggestionScore };
const aff = byAuthor.get(u.id) || { score: 0, common: [], reasons: [] };
const tribeMate = aff.reasons.includes('tribe');
if (aff.score < MIN_SUGGESTION_AFFINITY && !n.followsMe && !tribeMate) return null;
const social = (n.followsMe ? 0.15 : 0) + (tribeMate ? 0.1 : 0);
const activityBonus = u.lastActivityBucket === 'green' ? 0.05 : (u.lastActivityBucket === 'orange' ? 0.02 : 0);
const suggestionScore = Math.min(1, aff.score + social + activityBonus);
const karmaScore = karmaOf.get(u.id) || 0;
return { user: u, rel: n, karmaScore, commonSkills: aff.common, reasons: aff.reasons, suggestionScore };
})
);
const candidates = rels.filter(Boolean).filter(x => x.suggestionScore > 0);
const candidates = rels.filter(Boolean);
const enriched = candidates.map(x => ({
...x.user,
karmaScore: x.karmaScore,
followsYou: x.rel.followsMe,
commonSkills: x.commonSkills,
reasons: x.reasons,
mutualCount: x.rel.followsMe ? 1 : 0,
suggestionScore: x.suggestionScore
suggestionScore: x.suggestionScore,
affinity: x.suggestionScore
}));
const unique = Array.from(new Map(enriched.map(u => [u.id, u])).values());
return unique.sort((a, b) =>
(b.suggestionScore || 0) - (a.suggestionScore || 0) ||
(b.karmaScore || 0) - (a.karmaScore || 0) ||
(b.lastActivityTs || 0) - (a.lastActivityTs || 0)
);
).slice(0, MAX_SUGGESTED);
}
if (filter === 'CVs') {
const records = await new Promise((res, rej) => {
pull(
ssbClient.createLogStream({ limit: logLimit, reverse: true}),
pull.filter(msg =>
msg.value.content?.type === 'curriculum' &&
msg.value.content?.type !== 'tombstone'
),
pull.collect((err, msgs) => err ? rej(err) : res(msgs))
);
});
const records = (await readTyped(ssbClient, ['curriculum'], { limit: logLimit })).filter(msg => msg.value.content?.type === 'curriculum').reverse();
let cvs = records.map(r => r.value.content);
cvs = Array.from(new Map(cvs.map(u => [u.author, u])).values());
if (filter === 'CVs') {
const profiles = publicMode ? null : await about.profiles().catch(() => null);
let out = await Promise.all(cvs.map(async c => {
const photo = await fetchUserImageUrl(c.author, 256);
const photo = profiles ? imageUrlFrom((profiles.get(c.author) || {}).image, 256) : await fetchUserImageUrl(c.author, 256);
const lastActivityTs = await getLastActivityTimestamp(c.author);
const { bucket, range } = bucketLastActivity(lastActivityTs);
const base = this._normalizeCurriculum(c, photo);
@ -308,6 +298,7 @@ module.exports = ({ cooler }) => {
name: c.name,
description: c.description,
photo,
pdf: c.pdf || null,
skills: [
...(c.personalSkills || []),
...(c.oasisSkills || []),
@ -324,6 +315,14 @@ module.exports = ({ cooler }) => {
};
},
async listSupporters(feedId) {
const ssbClient = await openSsb();
const graph = await new Promise((resolve) => ssbClient.friends.graph((err, g) => resolve(err ? {} : (g || {}))));
return Object.entries(graph)
.filter(([source, edges]) => source !== feedId && edges && (edges[feedId] === true || Number(edges[feedId]) > 0))
.map(([source]) => source);
},
async getLatestAboutById(id) {
const ssbClient = await openSsb();
const records = await new Promise((res, rej) => {
@ -369,21 +368,8 @@ module.exports = ({ cooler }) => {
const isOwner = viewer === target;
const arr = (v) => Array.isArray(v) ? v : [];
const up = (v) => String(v || '').toUpperCase();
const COUNTED = new Set(['post','event','task','forum','tribe','market','job','housing','project','industry','shop','image','video','audio','document','bookmark','transfer','map']);
const accessible = (type, c) => {
if (c.encryptedPayload) return false;
switch (type) {
case 'task': return up(c.isPublic) !== 'PRIVATE' || isOwner || arr(c.assignees).includes(viewer);
case 'event': return String(c.isPublic || '').toLowerCase() !== 'private' || isOwner || arr(c.attendees).includes(viewer);
case 'forum': return c.isPrivate !== true || isOwner;
case 'job': return up(c.visibility) !== 'HIDDEN' || isOwner || arr(c.subscribers).includes(viewer);
case 'housing': return up(c.visibility) !== 'HIDDEN' || isOwner;
case 'market': return up(c.visibility) !== 'HIDDEN' || isOwner;
case 'shop': return up(c.visibility) !== 'CLOSED' || isOwner;
case 'tribe': { const st = up(c.status); return !(st === 'PRIVATE' || st === 'INVITE-ONLY') || isOwner || arr(c.members).includes(viewer); }
default: return true;
}
};
const COUNTED = new Set(['post','event','task','forum','market','job','housing','project','industry','shop','image','video','audio','document','bookmark','transfer','map']);
const accessible = (type, c) => isOwner || isContentVisibleTo(type, c, viewer, target);
const counts = {};
await new Promise((resolve) => {
pull(
@ -399,6 +385,13 @@ module.exports = ({ cooler }) => {
}, () => resolve())
);
});
if (tribesModel) {
try {
const visible = await tribesModel.listTribesForViewer(viewer);
const n = visible.filter(t => String(t.author) === String(target)).length;
if (n > 0) counts.tribe = n;
} catch (_) {}
}
return counts;
},
@ -435,16 +428,7 @@ module.exports = ({ cooler }) => {
addAll(tokenize(job.requirements));
if (keywords.size === 0) return [];
const records = await new Promise((res, rej) => {
pull(
ssbClient.createLogStream({ limit: logLimit, reverse: true }),
pull.filter(msg =>
msg.value?.content?.type === 'curriculum' &&
msg.value?.content?.type !== 'tombstone'
),
pull.collect((err, msgs) => err ? rej(err) : res(msgs))
);
});
const records = (await readTyped(ssbClient, ['curriculum'], { limit: logLimit })).filter(msg => msg.value?.content?.type === 'curriculum').reverse();
let cvs = records.map(r => r.value.content);
cvs = Array.from(new Map(cvs.map(u => [u.author, u])).values());
cvs = cvs.filter(c => String(c.visibility || 'PUBLIC').toUpperCase() !== 'HIDDEN');

129
src/models/jobs_model.js
View file

@ -2,9 +2,12 @@ const pull = require("../server/node_modules/pull-stream")
const moment = require("../server/node_modules/moment")
const { getConfig } = require("../configs/config-manager.js")
const { buildValidatedTombstoneSet } = require('./tombstone_validator');
const { dedupeByPreferring } = require('../backend/dedupe')
const { dedupeByPreferring, versionGroups } = require('../backend/dedupe')
const { readTyped, memoIndex } = require('./typed_log')
const logLimit = getConfig().ssbLogStream?.limit || 1000
const JOB_TYPES = ["job", "job_sub", "tombstone"]
const norm = (s) => String(s || "").trim().toLowerCase()
const toNum = (v) => {
const n = parseFloat(String(v ?? "").replace(",", "."))
@ -40,20 +43,12 @@ module.exports = ({ cooler, tribeCrypto }) => {
const openSsb = async () => { if (!ssb) ssb = await cooler.open(); return ssb }
const readAll = async (ssbClient) =>
new Promise((resolve, reject) =>
pull(
ssbClient.createLogStream({ limit: logLimit }),
pull.collect((err, msgs) => err ? reject(err) : resolve(msgs))
)
)
readTyped(ssbClient, JOB_TYPES, { limit: logLimit, withWindow: true, withPrivate: true })
const buildIndex = (messages, ssbClient) => {
const tomb = new Set()
const tomb = buildValidatedTombstoneSet(messages)
const jobNodes = new Map()
const parent = new Map()
const child = new Map()
const naiveReplaces = new Map()
const jobSubLatest = new Map()
const subMsgs = []
for (const m of messages) {
const key = m.key
@ -61,65 +56,19 @@ module.exports = ({ cooler, tribeCrypto }) => {
const c = v.content
if (!c) continue
if (c.type === "tombstone" && c.target) {
tomb.add(c.target)
continue
}
if (c.type === "job") {
jobNodes.set(key, { key, ts: v.timestamp || m.timestamp || 0, c, author: v.author })
if (c.replaces) naiveReplaces.set(key, c.replaces)
jobNodes.set(key, { key, ts: v.timestamp || m.timestamp || 0, seq: v.sequence || 0, c, author: v.author })
continue
}
if (c.type === "job_sub" && c.jobId) {
const author = v.author
if (!author) continue
const ts = v.timestamp || m.timestamp || 0
const jobId = c.jobId
const k = `${jobId}::${author}`
const prev = jobSubLatest.get(k)
if (!prev || ts >= prev.ts) jobSubLatest.set(k, { ts, value: !!c.value, author, jobId })
subMsgs.push({ ts: v.timestamp || m.timestamp || 0, value: !!c.value, author, jobId: c.jobId })
continue
}
}
for (const [key, replacesId] of naiveReplaces.entries()) {
const node = jobNodes.get(key)
if (!node) continue
const orig = jobNodes.get(replacesId)
if (!orig) continue
if (String(orig.author) !== String(node.author)) { jobNodes.delete(key); continue }
parent.set(key, replacesId)
child.set(replacesId, key)
}
const rootOf = (id) => {
let cur = id
while (parent.has(cur)) cur = parent.get(cur)
return cur
}
const roots = new Set()
for (const id of jobNodes.keys()) roots.add(rootOf(id))
const tipOf = (id) => {
let cur = id
while (child.has(cur)) cur = child.get(cur)
return cur
}
const tipByRoot = new Map()
for (const r of roots) tipByRoot.set(r, tipOf(r))
const subsByJob = new Map()
for (const { jobId, author, value } of jobSubLatest.values()) {
if (!subsByJob.has(jobId)) subsByJob.set(jobId, new Set())
const set = subsByJob.get(jobId)
if (value) set.add(author)
else set.delete(author)
}
if (ssbClient) {
for (const m of messages) {
if (typeof m.value?.content !== 'string') continue
@ -130,16 +79,30 @@ module.exports = ({ cooler, tribeCrypto }) => {
if (c.type !== 'job_sub' || !c.jobId) continue
const author = dec.value.author
if (!author) continue
const ts = dec.value.timestamp || m.timestamp || 0
const jobId = c.jobId
const k = `${jobId}::${author}`
const prev = jobSubLatest.get(k)
if (!prev || ts >= prev.ts) jobSubLatest.set(k, { ts, value: !!c.value, author, jobId })
subMsgs.push({ ts: dec.value.timestamp || m.timestamp || 0, value: !!c.value, author, jobId: c.jobId })
} catch {}
}
}
return { tomb, jobNodes, parent, child, rootOf, tipOf, tipByRoot, subsByJob }
const vg = versionGroups(Array.from(jobNodes.values()).map(n => ({ key: n.key, author: n.author, seq: n.seq, ts: n.ts, replaces: typeof n.c.replaces === "string" ? n.c.replaces : null, claimedAuthor: typeof n.c.author === "string" ? n.c.author : null })))
const jobSubLatest = new Map()
for (const sub of subMsgs) {
const jobId = vg.rootOf(sub.jobId)
const k = `${jobId}::${sub.author}`
const prev = jobSubLatest.get(k)
if (!prev || sub.ts >= prev.ts) jobSubLatest.set(k, { ...sub, jobId })
}
const subsByJob = new Map()
for (const { jobId, author, value } of jobSubLatest.values()) {
if (!subsByJob.has(jobId)) subsByJob.set(jobId, new Set())
const set = subsByJob.get(jobId)
if (value) set.add(author)
else set.delete(author)
}
return { tomb, jobNodes, vg, subsByJob }
}
const buildJobObject = (node, rootId, subscribers) => {
@ -255,26 +218,20 @@ module.exports = ({ cooler, tribeCrypto }) => {
async resolveCurrentId(jobId) {
const ssbClient = await openSsb()
const messages = await readAll(ssbClient)
const { tomb, child } = buildIndex(messages, ssbClient)
const { tomb, vg } = buildIndex(messages, ssbClient)
let cur = jobId
while (child.has(cur)) cur = child.get(cur)
if (tomb.has(cur)) throw new Error("Job not found")
const cur = vg.tipOf(jobId)
if (tomb.has(cur) || vg.isForeignCopy(vg.rootOf(cur))) throw new Error("Job not found")
return cur
},
async resolveRootId(jobId) {
const ssbClient = await openSsb()
const messages = await readAll(ssbClient)
const { tomb, parent, child } = buildIndex(messages, ssbClient)
const { tomb, vg } = buildIndex(messages, ssbClient)
let tip = jobId
while (child.has(tip)) tip = child.get(tip)
if (tomb.has(tip)) throw new Error("Job not found")
let root = tip
while (parent.has(root)) root = parent.get(root)
return root
if (tomb.has(vg.tipOf(jobId)) || vg.isForeignCopy(vg.rootOf(jobId))) throw new Error("Job not found")
return vg.rootOf(jobId)
},
async updateJob(id, jobData) {
@ -468,7 +425,8 @@ module.exports = ({ cooler, tribeCrypto }) => {
const idx = buildIndex(messages, ssbClient)
const jobs = []
for (const [rootId, tipId] of idx.tipByRoot.entries()) {
for (const rootId of idx.vg.roots) {
const tipId = idx.vg.tipOf(rootId)
if (idx.tomb.has(tipId)) continue
const node = idx.jobNodes.get(tipId)
if (!node) continue
@ -523,14 +481,11 @@ module.exports = ({ cooler, tribeCrypto }) => {
const viewer = viewerId || ssbClient.id
const messages = await readAll(ssbClient)
const idx = buildIndex(messages, ssbClient)
const idx = memoIndex('jobs', messages, (m) => buildIndex(m, ssbClient))
let tipId = id
while (idx.child.has(tipId)) tipId = idx.child.get(tipId)
if (idx.tomb.has(tipId)) throw new Error("Job not found")
let rootId = tipId
while (idx.parent.has(rootId)) rootId = idx.parent.get(rootId)
const tipId = idx.vg.tipOf(id)
const rootId = idx.vg.rootOf(id)
if (idx.tomb.has(tipId) || idx.vg.isForeignCopy(rootId)) throw new Error("Job not found")
const gate = (job) => {
if (job.visibility === "HIDDEN" && job.author !== viewer) throw new Error("Job not found")
@ -540,7 +495,7 @@ module.exports = ({ cooler, tribeCrypto }) => {
const node = idx.jobNodes.get(tipId)
if (!node) {
const msg = await new Promise((r, j) => ssbClient.get(tipId, (e, m) => e ? j(e) : r(m)))
if (!msg || !msg.content) throw new Error("Job not found")
if (!msg || !msg.content || msg.content.type !== "job" || (msg.content.author && msg.content.author !== msg.author)) throw new Error("Job not found")
const tmpNode = { key: tipId, ts: msg.timestamp || 0, c: msg.content, author: msg.author }
const subsSet = idx.subsByJob.get(rootId) || new Set()
const subs = Array.from(subsSet)

235
src/models/larp_model.js
View file

@ -1,4 +1,5 @@
const pull = require('../server/node_modules/pull-stream');
const { readTyped, requestScope } = require('./typed_log');
const fs = require('fs');
const path = require('path');
@ -10,6 +11,7 @@ const HOUSE_KEYS = ['academia','solaris','arrakis','terraverde','unsystem','dogm
const VALID_KEY = (k) => HOUSE_KEYS.includes(String(k || '').toLowerCase());
const TEST_COOLDOWN_MS = 30 * 24 * 60 * 60 * 1000;
const ACADEMIA_GRACE_MS = 10 * 60 * 1000;
const PROFILE_QUESTIONS = [
{
@ -173,6 +175,12 @@ function getGoverningPeriodId(now = new Date()) {
module.exports = ({ cooler, tribesModel, tribeCrypto }) => {
let ssb;
const openSsb = async () => { if (!ssb) ssb = await cooler.open(); return ssb; };
const startedAt = Date.now();
let academiaRetry = null;
const followsSomeone = async (client, me) => {
const contacts = await readTyped(client, ['contact']).catch(() => []);
return contacts.some((m) => m && m.value && m.value.author === me && m.value.content && m.value.content.following === true);
};
const houseTribeTag = (houseKey) => {
const h = HOUSES[houseKey];
@ -192,7 +200,7 @@ module.exports = ({ cooler, tribesModel, tribeCrypto }) => {
return tags.includes(tag) && members.includes(me);
});
if (!candidates.length) return null;
candidates.sort((a, b) => new Date(a.createdAt || 0).getTime() - new Date(b.createdAt || 0).getTime());
candidates.sort((a, b) => (new Date(a.createdAt || 0).getTime() - new Date(b.createdAt || 0).getTime()) || String(a.id || '').localeCompare(String(b.id || '')));
return candidates[0];
}
@ -206,78 +214,59 @@ module.exports = ({ cooler, tribesModel, tribeCrypto }) => {
return tags.includes(tag);
});
if (!candidates.length) return null;
candidates.sort((a, b) => new Date(a.createdAt || 0).getTime() - new Date(b.createdAt || 0).getTime());
candidates.sort((a, b) => (new Date(a.createdAt || 0).getTime() - new Date(b.createdAt || 0).getTime()) || String(a.id || '').localeCompare(String(b.id || '')));
return candidates[0];
}
const ANCHOR_TYPES = ['larpHouseTribeAnchor', 'larpHouseTribeAnchorTombstone', 'tombstone'];
const authorOfMsg = (client, key) => new Promise((resolve) => client.get(key, (err, v) => resolve(err || !v ? null : v.author || null)));
async function findEarliestHouseAnchor(houseKey) {
if (!VALID_KEY(houseKey)) return null;
const client = await openSsb();
return new Promise((resolve) => {
const anchors = [];
const anchorTombstones = [];
const tribeTombstones = new Map();
const msgAuthorByKey = new Map();
pull(
client.createLogStream(),
pull.drain((m) => {
if (!m || !m.value) return;
const author = m.value.author;
if (m.key && author) msgAuthorByKey.set(m.key, author);
const c = m.value.content;
if (!c || typeof c !== 'object') return;
if (c.type === 'larpHouseTribeAnchor') {
if (c.house !== houseKey) return;
if (typeof c.tribeRootId !== 'string') return;
const tribeTs = Number(Date.parse(c.tribeCreatedAt || '')) || m.value.timestamp || 0;
anchors.push({ tribeRootId: c.tribeRootId, anchorAuthor: author, tribeTs });
} else if (c.type === 'larpHouseTribeAnchorTombstone') {
if (c.house !== houseKey) return;
if (typeof c.tribeRootId !== 'string') return;
anchorTombstones.push({ tribeRootId: c.tribeRootId, tombstoneAuthor: author });
} else if (c.type === 'tombstone' && typeof c.target === 'string') {
tribeTombstones.set(c.target, author);
}
}, () => {
const killedAnchorPairs = new Set();
for (const t of anchorTombstones) {
killedAnchorPairs.add(`${t.tribeRootId}|${t.tombstoneAuthor}`);
}
const deadTribes = new Set();
for (const [target, tombAuthor] of tribeTombstones) {
const tribeAuthor = msgAuthorByKey.get(target);
if (tribeAuthor && tribeAuthor === tombAuthor) deadTribes.add(target);
}
const live = anchors.filter(a =>
!killedAnchorPairs.has(`${a.tribeRootId}|${a.anchorAuthor}`) &&
!deadTribes.has(a.tribeRootId)
);
if (!live.length) return resolve(null);
live.sort((a, b) => a.tribeTs - b.tribeTs);
const first = live[0];
resolve({ tribeRootId: first.tribeRootId, author: first.anchorAuthor, tribeTs: first.tribeTs });
})
);
});
const msgs = await readTyped(client, ANCHOR_TYPES);
const anchors = [];
const killedAnchorPairs = new Set();
const tombAuthors = new Map();
for (const m of msgs) {
const c = m && m.value && m.value.content;
if (!c || typeof c !== 'object') continue;
const author = m.value.author;
if (c.type === 'larpHouseTribeAnchor') {
if (c.house !== houseKey || typeof c.tribeRootId !== 'string') continue;
const tribeTs = Number(Date.parse(c.tribeCreatedAt || '')) || m.value.timestamp || 0;
anchors.push({ tribeRootId: c.tribeRootId, anchorAuthor: author, tribeTs });
} else if (c.type === 'larpHouseTribeAnchorTombstone') {
if (c.house !== houseKey || typeof c.tribeRootId !== 'string') continue;
killedAnchorPairs.add(`${c.tribeRootId}|${author}`);
} else if (c.type === 'tombstone' && typeof c.target === 'string') {
if (!tombAuthors.has(c.target)) tombAuthors.set(c.target, new Set());
tombAuthors.get(c.target).add(author);
}
}
const candidates = anchors.filter(a => !killedAnchorPairs.has(`${a.tribeRootId}|${a.anchorAuthor}`));
const deadTribes = new Set();
await Promise.all([...new Set(candidates.map(a => a.tribeRootId).filter(t => tombAuthors.has(t)))].map(async (t) => {
const tribeAuthor = await authorOfMsg(client, t);
if (tribeAuthor && tombAuthors.get(t).has(tribeAuthor)) deadTribes.add(t);
}));
const live = candidates.filter(a => !deadTribes.has(a.tribeRootId));
if (!live.length) return null;
live.sort((a, b) => a.tribeTs - b.tribeTs);
const first = live[0];
return { tribeRootId: first.tribeRootId, author: first.anchorAuthor, tribeTs: first.tribeTs };
}
async function findHouseAnchorByTribe(houseKey, tribeRootId) {
if (!VALID_KEY(houseKey) || !tribeRootId) return null;
const client = await openSsb();
return new Promise((resolve) => {
let hit = null;
pull(
client.createLogStream(),
pull.drain((m) => {
if (hit) return;
const c = m && m.value && m.value.content;
if (!c || c.type !== 'larpHouseTribeAnchor') return;
if (c.house !== houseKey) return;
if (c.tribeRootId !== tribeRootId) return;
hit = { author: m.value.author, ts: m.value.timestamp || 0 };
}, () => resolve(hit))
);
});
const msgs = await readTyped(client, ANCHOR_TYPES);
for (const m of msgs) {
const c = m && m.value && m.value.content;
if (!c || c.type !== 'larpHouseTribeAnchor' || c.house !== houseKey || c.tribeRootId !== tribeRootId) continue;
return { author: m.value.author, ts: m.value.timestamp || 0 };
}
return null;
}
async function publishHouseTribeAnchor(houseKey, tribeRootId, tribeCreatedAt) {
@ -332,7 +321,7 @@ module.exports = ({ cooler, tribesModel, tribeCrypto }) => {
const canonical = await findAnyHouseTribe(houseKey).catch(() => null);
const canonRoot = canonical ? await tribesModel.getRootId(canonical.id).catch(() => canonical.id) : null;
const myTribes = await listMyHouseTribes(houseKey);
myTribes.sort((a, b) => a.createdAtTs - b.createdAtTs);
myTribes.sort((a, b) => (a.createdAtTs - b.createdAtTs) || String(a.rootId || '').localeCompare(String(b.rootId || '')));
const keepRoot = canonRoot || (myTribes[0] && myTribes[0].rootId);
for (const t of myTribes) {
if (t.rootId === keepRoot) continue;
@ -345,6 +334,13 @@ module.exports = ({ cooler, tribesModel, tribeCrypto }) => {
await redeemPendingAutoInvites().catch(() => {});
return await findMyHouseTribe(houseKey) || canonical;
}
if (Date.now() - startedAt < ACADEMIA_GRACE_MS && await followsSomeone(client, me)) {
if (!academiaRetry) {
academiaRetry = setTimeout(() => { academiaRetry = null; enqueue(runCatchup); }, ACADEMIA_GRACE_MS - (Date.now() - startedAt));
if (academiaRetry.unref) academiaRetry.unref();
}
return null;
}
const house = HOUSES[houseKey] || {};
const tag = houseTribeTag(houseKey);
try {
@ -460,6 +456,7 @@ module.exports = ({ cooler, tribesModel, tribeCrypto }) => {
joinedAt: new Date().toISOString()
}, (err, msg) => err ? reject(err) : resolve(msg));
});
forgetMemberships();
if (previousHouse && previousHouse !== houseKey) {
await leaveMyHouseTribe(previousHouse).catch(() => {});
}
@ -471,53 +468,45 @@ module.exports = ({ cooler, tribesModel, tribeCrypto }) => {
async function getUserHouse(feedId) {
const client = await openSsb();
const target = feedId || client.id;
return new Promise((resolve) => {
let latest = null;
let latestTs = 0;
pull(
client.createUserStream({ id: target, reverse: true }),
pull.drain((m) => {
const c = m && m.value && m.value.content;
if (!c) return;
const ts = m.value.timestamp || 0;
if (c.type === 'larpJoinHouse' && VALID_KEY(c.house)) {
if (ts > latestTs) { latestTs = ts; latest = c.house; }
} else if (c.type === 'larpLeaveLarp') {
if (ts > latestTs) { latestTs = ts; latest = null; }
}
}, () => resolve(latest))
);
});
return (await listAllMemberships()).get(target) || null;
}
async function listAllMemberships() {
async function readMemberships() {
const client = await openSsb();
return new Promise((resolve) => {
const byAuthor = new Map();
pull(
client.createLogStream({ reverse: true }),
pull.drain((m) => {
const author = m && m.value && m.value.author;
if (!author) return;
const c = m.value.content;
if (!c) return;
const ts = m.value.timestamp || 0;
if (c.type === 'larpJoinHouse' && VALID_KEY(c.house)) {
const prev = byAuthor.get(author);
if (!prev || ts > prev.ts) byAuthor.set(author, { house: c.house, ts });
} else if (c.type === 'larpLeaveLarp') {
const prev = byAuthor.get(author);
if (!prev || ts > prev.ts) byAuthor.set(author, { house: null, ts });
}
}, () => {
const result = new Map();
for (const [a, v] of byAuthor.entries()) {
if (v.house) result.set(a, v.house);
}
resolve(result);
})
);
});
const msgs = await readTyped(client, ['larpJoinHouse', 'larpLeaveLarp'], { limit: 0 });
const byAuthor = new Map();
for (const m of msgs) {
const author = m && m.value && m.value.author;
if (!author) continue;
const c = m.value.content;
if (!c) continue;
const ts = m.value.timestamp || 0;
const seq = Number(m.value.sequence) || 0;
const prev = byAuthor.get(author);
const newer = !prev || ts > prev.ts || (ts === prev.ts && seq > prev.seq);
if (c.type === 'larpJoinHouse' && VALID_KEY(c.house)) {
if (newer) byAuthor.set(author, { house: c.house, ts, seq });
} else if (c.type === 'larpLeaveLarp') {
if (newer) byAuthor.set(author, { house: null, ts, seq });
}
}
const result = new Map();
for (const [a, v] of byAuthor.entries()) {
if (v.house) result.set(a, v.house);
}
return result;
}
const forgetMemberships = () => {
const store = requestScope.getStore();
if (store) delete store.larpMemberships;
};
async function listAllMemberships() {
const store = requestScope.getStore();
if (!store) return readMemberships();
if (!store.larpMemberships) store.larpMemberships = readMemberships();
return store.larpMemberships;
}
async function publishLeaveLarp() {
@ -530,6 +519,7 @@ module.exports = ({ cooler, tribesModel, tribeCrypto }) => {
leftAt: new Date().toISOString()
}, (err, msg) => err ? reject(err) : resolve(msg));
});
forgetMemberships();
if (previousHouse) {
await leaveMyHouseTribe(previousHouse).catch(() => {});
}
@ -585,10 +575,16 @@ module.exports = ({ cooler, tribesModel, tribeCrypto }) => {
const memberships = await listAllMemberships();
return new Promise((resolve) => {
const posts = [];
const tombs = new Map();
pull(
client.createLogStream({ reverse: true }),
pull.drain((m) => {
const c = m && m.value && m.value.content;
if (c && c.type === 'tombstone' && typeof c.target === 'string') {
if (!tombs.has(c.target)) tombs.set(c.target, new Set());
tombs.get(c.target).add(m.value.author);
return;
}
if (!c || c.type !== 'larpHousePost') return;
if (c.house !== houseKey) return;
const author = m.value.author;
@ -602,8 +598,9 @@ module.exports = ({ cooler, tribesModel, tribeCrypto }) => {
ts: m.value.timestamp || 0
});
}, () => {
posts.sort((a, b) => b.ts - a.ts);
resolve(posts);
const shown = posts.filter(post => !(tombs.get(post.id) && tombs.get(post.id).has(post.author)));
shown.sort((a, b) => b.ts - a.ts);
resolve(shown);
})
);
});
@ -770,14 +767,20 @@ module.exports = ({ cooler, tribesModel, tribeCrypto }) => {
const ssbKeys = require('../server/node_modules/ssb-keys');
const config = require('../server/ssb_config');
const codes = [];
const sealedOnly = client.private && typeof client.private.read === 'function';
await new Promise((resolve) => {
pull(
client.createLogStream({ reverse: true, limit: 2000 }),
sealedOnly ? client.private.read({ reverse: true }) : client.createLogStream({ reverse: true, limit: 2000 }),
pull.drain((m) => {
const c = m && m.value && m.value.content;
if (typeof c !== 'string' || !c.endsWith('.box')) return;
let decoded;
try { decoded = ssbKeys.unbox(c, config.keys); } catch (_) { return; }
if (typeof c === 'string') {
if (!c.endsWith('.box')) return;
try { decoded = ssbKeys.unbox(c, config.keys); } catch (_) { return; }
} else {
if (!m.value.private) return;
decoded = c;
}
if (!decoded) return;
if (typeof decoded === 'string') {
try { decoded = JSON.parse(decoded); } catch (_) { return; }
@ -832,6 +835,13 @@ module.exports = ({ cooler, tribesModel, tribeCrypto }) => {
await issueAutoInvitesForMyHouse().catch(() => {});
return;
}
if (typeof c === 'object' && c.type === 'tribe' && Array.isArray(c.tags) && c.tags.includes(houseTribeTag('academia'))) {
if (m.value.author === me) return;
const myHouse = await getUserHouse(me).catch(() => null);
if (myHouse !== 'academia') return;
await ensureHouseTribe('academia').catch(() => {});
return;
}
if (typeof c === 'object' && c.type === 'larpHouseTribeAnchor' && VALID_KEY(c.house)) {
if (m.value.author === me) return;
const myHouse = await getUserHouse(me).catch(() => null);
@ -931,6 +941,7 @@ module.exports = ({ cooler, tribesModel, tribeCrypto }) => {
publishJoin,
publishLeaveLarp,
getUserHouse,
listAllMemberships,
listHousesWithCounts,
getMembersOfHouse,
publishHousePost,

97
src/models/legacy_model.js
View file

@ -1,97 +0,0 @@
const fs = require('fs');
const path = require('path');
const crypto = require('crypto');
const os = require('os');
const MAGIC = Buffer.from('OASIS1');
const SALT_LEN = 16;
const IV_LEN = 12;
const TAG_LEN = 16;
function normalizePassword(password) {
if (password && typeof password === 'object' && password.password) password = password.password;
return String(password || '');
}
function deriveKey(password, salt) {
return crypto.scryptSync(password, salt, 32);
}
function encryptBuffer(plaintext, password) {
const salt = crypto.randomBytes(SALT_LEN);
const iv = crypto.randomBytes(IV_LEN);
const key = deriveKey(password, salt);
const cipher = crypto.createCipheriv('aes-256-gcm', key, iv);
const ciphertext = Buffer.concat([cipher.update(plaintext), cipher.final()]);
const tag = cipher.getAuthTag();
return Buffer.concat([MAGIC, salt, iv, tag, ciphertext]);
}
function decryptBuffer(data, password) {
if (!Buffer.isBuffer(data) || data.length < MAGIC.length + SALT_LEN + IV_LEN + TAG_LEN) {
throw new Error('Unrecognized or corrupt backup file.');
}
if (!data.slice(0, MAGIC.length).equals(MAGIC)) {
throw new Error('Unrecognized or corrupt backup file.');
}
let o = MAGIC.length;
const salt = data.slice(o, o += SALT_LEN);
const iv = data.slice(o, o += IV_LEN);
const tag = data.slice(o, o += TAG_LEN);
const ciphertext = data.slice(o);
const key = deriveKey(password, salt);
const decipher = crypto.createDecipheriv('aes-256-gcm', key, iv);
decipher.setAuthTag(tag);
return Buffer.concat([decipher.update(ciphertext), decipher.final()]);
}
module.exports = {
exportData: async (password) => {
try {
const pw = normalizePassword(password);
// identidad de la cuenta activa
const secretFilePath = path.join(require('../server/ssb_config').path, 'secret');
if (!fs.existsSync(secretFilePath)) {
throw new Error(".ssb/secret file doesn't exist");
}
const original = fs.readFileSync(secretFilePath);
const encrypted = encryptBuffer(original, pw);
const roundtrip = decryptBuffer(encrypted, pw);
if (!roundtrip.equals(original)) {
throw new Error('Backup verification failed; nothing was exported.');
}
return { filename: 'oasis.enc', data: encrypted };
} catch (error) {
throw new Error("Error exporting data: " + error.message);
}
},
importData: async ({ filePath, password }) => {
try {
const pw = normalizePassword(password);
if (!fs.existsSync(filePath)) {
throw new Error('Encrypted file not found.');
}
const data = fs.readFileSync(filePath);
let decrypted;
try {
decrypted = decryptBuffer(data, pw);
} catch (_) {
throw new Error('Wrong password or corrupt backup file.');
}
// identidad de la cuenta activa
const ssbDir = require('../server/ssb_config').path;
fs.mkdirSync(ssbDir, { recursive: true });
const secretPath = path.join(ssbDir, 'secret');
if (fs.existsSync(secretPath)) {
fs.copyFileSync(secretPath, path.join(ssbDir, 'secret.bak-' + new Date().toISOString().replace(/[:.]/g, '-')));
}
const tmpPath = path.join(ssbDir, 'secret.tmp-' + process.pid);
fs.writeFileSync(tmpPath, decrypted, { mode: 0o600 });
fs.renameSync(tmpPath, secretPath);
try { fs.unlinkSync(filePath); } catch (_) {}
return secretPath;
} catch (error) {
throw new Error("Error importing data: " + error.message);
}
}
};

390
src/models/logistics_model.js Normal file
View file

@ -0,0 +1,390 @@
const { readTyped, memoIndex } = require('./typed_log');
const categories = require('../backend/opinion_categories');
const { buildValidatedTombstoneSet } = require('./tombstone_validator');
const { getConfig } = require('../configs/config-manager.js');
const logLimit = getConfig().ssbLogStream?.limit || 1000;
const ROUTE_TYPE = 'logisticsRoute';
const RATING_TYPE = 'logisticsRating';
const BOOKING_TYPE = 'logistics-booking';
const BOOKING_STATUS_TYPE = 'logistics-booking-status';
const OPINION_TYPE = 'logisticsOpinion';
const PUBLIC_TYPES = [ROUTE_TYPE, RATING_TYPE, OPINION_TYPE, 'tombstone'];
const KINDS = ['TRIP', 'SHIPMENT'];
const MODES = ['OFFER', 'REQUEST'];
const PRICE_TYPES = ['FREE', 'ECO', 'TIME'];
const RECURRENCES = ['NONE', 'DAILY', 'WEEKLY', 'MONTHLY'];
const STATUSES = ['OPEN', 'CLOSED'];
const BOOKING_STATUSES = ['PENDING', 'CONFIRMED', 'REJECTED', 'CANCELLED', 'DELIVERED'];
const OWNER_BOOKING_STATUSES = ['CONFIRMED', 'REJECTED', 'DELIVERED'];
const RECENT_MS = 24 * 60 * 60 * 1000;
const MAX_ZONES = 8;
const safeText = (v) => String(v == null ? '' : v).trim();
const normU = (v) => String(v || '').trim().toUpperCase();
const pick = (list, v, fallback) => (list.includes(normU(v)) ? normU(v) : fallback);
const normalizeTags = (raw) => {
if (!raw) return [];
const arr = Array.isArray(raw) ? raw : String(raw).split(',');
return Array.from(new Set(arr.map(t => String(t || '').trim()).filter(Boolean)));
};
const toIso = (v) => { const t = Date.parse(v || ''); return Number.isFinite(t) ? new Date(t).toISOString() : ''; };
const num = (v, min = 0) => { const n = Number(v); return Number.isFinite(n) && n >= min ? n : 0; };
const MEDIA_RE = /!?\[(image|video):[^\]]*\]\((&[^)]+)\)/;
const mediaOf = (text) => { const m = safeText(text).match(MEDIA_RE); return m ? { kind: m[1], blobId: m[2] } : null; };
const zoneKey = (s) => safeText(s).toLowerCase();
module.exports = ({ cooler }) => {
let ssb;
const openSsb = async () => { if (!ssb) ssb = await cooler.open(); return ssb; };
const publish = (ssbClient, content) => new Promise((resolve, reject) => ssbClient.publish(content, (err, res) => err ? reject(err) : resolve(res)));
const publishPrivate = (ssbClient, content, recps) => new Promise((resolve, reject) => ssbClient.private.publish(content, recps, (err, res) => err ? reject(err) : resolve(res)));
const readPublic = async (ssbClient) => readTyped(ssbClient, PUBLIC_TYPES, { limit: logLimit });
const readPrivate = async (ssbClient) => {
const msgs = await readTyped(ssbClient, [BOOKING_TYPE, BOOKING_STATUS_TYPE], { limit: logLimit, withPrivate: true });
const bookings = [];
const statuses = [];
for (const m of msgs) {
const v = (m && m.value) || {};
if (v.private !== true) continue;
const c = v.content;
if (!c || typeof c !== 'object') continue;
const ts = v.timestamp || m.timestamp || 0;
if (c.type === BOOKING_TYPE && c.routeId) bookings.push({ id: m.key, author: v.author, ts, c });
else if (c.type === BOOKING_STATUS_TYPE && c.bookingId) statuses.push({ key: m.key, author: v.author, ts, c });
}
return { bookings, statuses };
};
const buildIndex = (messages) => {
const tomb = buildValidatedTombstoneSet(messages);
const nodes = new Map();
const parentOf = new Map();
const ratings = [];
const opinions = [];
for (const m of messages) {
const v = m.value || {};
const c = v.content;
if (!c || typeof c !== 'object') continue;
if (c.type === ROUTE_TYPE) {
nodes.set(m.key, { key: m.key, author: v.author, ts: v.timestamp || m.timestamp || 0, c });
if (typeof c.replaces === 'string') parentOf.set(m.key, c.replaces);
} else if (c.type === OPINION_TYPE && typeof c.target === 'string') {
opinions.push({ target: c.target, author: v.author, category: String(c.category || ''), ts: v.timestamp || 0 });
} else if (c.type === RATING_TYPE && typeof c.target === 'string') {
ratings.push({ key: m.key, target: c.target, author: v.author, ts: v.timestamp || 0, score: Math.min(5, Math.max(1, num(c.score, 1) || 1)), text: safeText(c.text), createdAt: c.createdAt || new Date(v.timestamp || 0).toISOString() });
}
}
const rootOf = (key) => {
const seen = new Set();
let cur = key;
while (parentOf.has(cur) && !seen.has(cur)) { seen.add(cur); cur = parentOf.get(cur); }
return cur;
};
const versionsByRoot = new Map();
for (const node of nodes.values()) {
const root = rootOf(node.key);
if (tomb.has(root)) continue;
if (!versionsByRoot.has(root)) versionsByRoot.set(root, []);
versionsByRoot.get(root).push(node);
}
const routes = new Map();
const now = Date.now();
for (const [root, versions] of versionsByRoot) {
versions.sort((a, b) => a.ts - b.ts);
const first = versions[0];
const own = versions.filter(v => v.author === first.author);
const tip = own[own.length - 1];
const c = tip.c;
const dateIso = toIso(c.date);
const dateTs = Date.parse(dateIso) || 0;
const recurrence = pick(RECURRENCES, c.recurrence, 'NONE');
const past = !!dateTs && dateTs < now && recurrence === 'NONE';
const status = pick(STATUSES, c.status, 'OPEN') === 'CLOSED' ? 'CLOSED' : (past ? 'PAST' : 'OPEN');
const byAuthor = new Map();
for (const r of ratings.filter(r => nodes.has(r.target) && rootOf(r.target) === root)) {
const prev = byAuthor.get(r.author);
if (!prev || r.ts >= prev.ts) byAuthor.set(r.author, r);
}
const routeRatings = Array.from(byAuthor.values()).sort((a, b) => b.ts - a.ts);
const ops = {};
const voters = [];
for (const op of opinions.filter(o => nodes.has(o.target) && rootOf(o.target) === root).sort((a, b) => a.ts - b.ts)) {
if (voters.includes(op.author) || !categories.includes(op.category)) continue;
voters.push(op.author);
ops[op.category] = (ops[op.category] || 0) + 1;
}
const ratingAvg = routeRatings.length ? Math.round((routeRatings.reduce((s, r) => s + r.score, 0) / routeRatings.length) * 10) / 10 : 0;
routes.set(root, {
id: root, rootId: root, tipId: tip.key,
kind: pick(KINDS, c.kind, 'TRIP'), mode: pick(MODES, c.mode, 'OFFER'),
title: safeText(c.title), description: safeText(c.description),
origin: safeText(c.origin), destination: safeText(c.destination), mapUrl: safeText(c.mapUrl),
date: dateIso, dateTs, recurrence,
seats: num(c.seats), seatsTaken: num(c.seatsTaken), size: safeText(c.size), weight: safeText(c.weight), media: mediaOf(c.description),
priceType: pick(PRICE_TYPES, c.priceType, 'FREE'), price: num(c.price),
orderRef: safeText(c.orderRef), tags: normalizeTags(c.tags),
status, closed: status !== 'OPEN',
author: first.author, createdAt: first.c.createdAt || new Date(first.ts).toISOString(),
updatedAt: c.updatedAt || new Date(tip.ts).toISOString(), ts: tip.ts,
ratings: routeRatings, ratingCount: routeRatings.length, ratingAvg,
opinions: ops, opinions_inhabitants: voters, opinionCount: voters.length
});
}
return { routes, rootOf, nodes };
};
const bookingsFor = (priv, routeId, me) => {
const statusById = new Map();
for (const s of priv.statuses) {
const prev = statusById.get(s.c.bookingId);
if (!prev || s.ts >= prev.ts) statusById.set(s.c.bookingId, { status: pick(BOOKING_STATUSES, s.c.status, 'PENDING'), ts: s.ts, by: s.author, receipt: safeText(s.c.receipt) });
}
return priv.bookings
.filter(b => String(b.c.routeId) === String(routeId))
.map(b => {
const st = statusById.get(b.id) || null;
return {
id: b.id, routeId: String(b.c.routeId), booker: b.author, owner: safeText(b.c.owner),
seats: num(b.c.seats), notes: safeText(b.c.notes), orderRef: safeText(b.c.orderRef),
createdAt: b.c.createdAt || new Date(b.ts).toISOString(), ts: b.ts,
status: st ? st.status : 'PENDING', statusAt: st ? new Date(st.ts).toISOString() : null, receipt: st ? st.receipt : '',
mine: b.author === me
};
})
.sort((a, b) => b.ts - a.ts);
};
const load = async () => {
const ssbClient = await openSsb();
const [pub, priv] = await Promise.all([readPublic(ssbClient), readPrivate(ssbClient)]);
return { ssbClient, idx: memoIndex('logistics', pub, buildIndex), priv };
};
const find = (idx, id) => {
const key = String(id || '');
if (idx.routes.has(key)) return idx.routes.get(key);
if (idx.nodes.has(key)) return idx.routes.get(idx.rootOf(key)) || null;
return null;
};
const decorate = (route, priv, me) => {
const bookings = bookingsFor(priv, route.id, me);
const confirmed = bookings.filter(b => b.status === 'CONFIRMED' || b.status === 'DELIVERED');
const taken = Math.max(route.seatsTaken || 0, confirmed.reduce((s, b) => s + (b.seats || 1), 0));
const myBooking = bookings.find(b => b.mine && b.status !== 'CANCELLED' && b.status !== 'REJECTED') || null;
const isOwner = route.author === me;
const participated = isOwner || confirmed.some(b => b.mine);
const myRating = route.ratings.find(r => r.author === me) || null;
return {
...route,
bookings: isOwner ? bookings : bookings.filter(b => b.mine),
bookingCount: bookings.filter(b => b.status !== 'CANCELLED' && b.status !== 'REJECTED').length,
confirmedCount: confirmed.length, seatsTaken: taken,
seatsLeft: route.kind === 'TRIP' && route.seats ? Math.max(0, route.seats - taken) : null,
myBooking, isOwner, participated,
canBook: !isOwner && !route.closed && !myBooking,
canRate: participated && route.closed && !myRating,
myRating,
lastActivityTs: Math.max(route.ts, ...bookings.map(b => b.ts), ...route.ratings.map(r => r.ts))
};
};
const routeContent = (userId, data, base = null) => ({
type: ROUTE_TYPE,
kind: pick(KINDS, data.kind, base ? base.kind : 'TRIP'),
mode: pick(MODES, data.mode, base ? base.mode : 'OFFER'),
title: safeText(data.title) || (base ? base.title : ''),
description: data.description !== undefined ? safeText(data.description) : (base ? base.description : ''),
origin: data.origin !== undefined ? safeText(data.origin) : (base ? base.origin : ''),
destination: data.destination !== undefined ? safeText(data.destination) : (base ? base.destination : ''),
mapUrl: data.mapUrl !== undefined ? safeText(data.mapUrl) : (base ? base.mapUrl : ''),
date: data.date !== undefined ? toIso(data.date) : (base ? base.date : ''),
recurrence: pick(RECURRENCES, data.recurrence, base ? base.recurrence : 'NONE'),
seats: data.seats !== undefined ? num(data.seats) : (base ? base.seats : 0),
size: data.size !== undefined ? safeText(data.size) : (base ? base.size : ''),
weight: data.weight !== undefined ? safeText(data.weight) : (base ? base.weight : ''),
priceType: pick(PRICE_TYPES, data.priceType, base ? base.priceType : 'FREE'),
price: data.price !== undefined ? num(data.price) : (base ? base.price : 0),
orderRef: data.orderRef !== undefined ? safeText(data.orderRef) : (base ? base.orderRef : ''),
tags: data.tags !== undefined ? normalizeTags(data.tags) : (base ? base.tags : []),
status: data.status !== undefined ? pick(STATUSES, data.status, 'OPEN') : (base ? (base.status === 'CLOSED' ? 'CLOSED' : 'OPEN') : 'OPEN'),
seatsTaken: data.seatsTaken !== undefined ? num(data.seatsTaken) : (base ? base.seatsTaken : 0),
author: userId,
createdAt: base ? base.createdAt : new Date().toISOString(),
updatedAt: new Date().toISOString()
});
return {
KINDS, MODES, PRICE_TYPES, RECURRENCES, STATUSES, BOOKING_STATUSES,
async createRoute(data = {}) {
const ssbClient = await openSsb();
const content = routeContent(ssbClient.id, data);
if (!content.title) throw new Error('Title is required');
if (!content.origin || !content.destination) throw new Error('Origin and destination are required');
if (!content.date) throw new Error('A valid date is required');
if (!(content.price > 0)) { content.price = 0; content.priceType = 'FREE'; } else if (content.priceType === 'FREE') content.priceType = 'ECO';
return publish(ssbClient, content);
},
async updateRoute(id, data = {}) {
const { ssbClient, idx } = await load();
const route = find(idx, id);
if (!route) throw new Error('Route not found');
if (route.author !== ssbClient.id) throw new Error('Only the author can update this route');
const content = { ...routeContent(ssbClient.id, data, route), replaces: route.tipId };
if (!(content.price > 0)) { content.price = 0; content.priceType = 'FREE'; } else if (content.priceType === 'FREE') content.priceType = 'ECO';
const res = await publish(ssbClient, content);
return { key: res.key, rootId: route.id };
},
async createOpinion(id, category) {
if (!categories.includes(category)) throw new Error('Invalid voting category');
const { ssbClient, idx } = await load();
const route = find(idx, id);
if (!route) throw new Error('Route not found');
if (route.opinions_inhabitants.includes(ssbClient.id)) throw new Error('Already voted');
return publish(ssbClient, { type: OPINION_TYPE, target: route.id, category, createdAt: new Date().toISOString() });
},
async closeRoute(id) { return this.updateRoute(id, { status: 'CLOSED' }); },
async reopenRoute(id) { return this.updateRoute(id, { status: 'OPEN' }); },
async deleteRoute(id) {
const { ssbClient, idx } = await load();
const route = find(idx, id);
if (!route) throw new Error('Route not found');
if (route.author !== ssbClient.id) throw new Error('Only the author can delete this route');
await publish(ssbClient, { type: 'tombstone', target: route.id, deletedAt: new Date().toISOString(), author: ssbClient.id });
return { key: route.id };
},
async book(id, { seats = 1, notes = '', orderRef = '' } = {}) {
const { ssbClient, idx, priv } = await load();
const me = ssbClient.id;
const route = find(idx, id);
if (!route) throw new Error('Route not found');
const view = decorate(route, priv, me);
if (view.isOwner) throw new Error('You cannot book your own route');
if (view.closed) throw new Error('This route is closed');
if (view.myBooking) throw new Error('You already booked this route');
const n = route.kind === 'TRIP' ? Math.max(1, Math.floor(num(seats, 1) || 1)) : 1;
if (view.seatsLeft !== null && n > view.seatsLeft) throw new Error('Not enough seats left');
const content = { type: BOOKING_TYPE, routeId: route.id, owner: route.author, seats: n, notes: safeText(notes), orderRef: safeText(orderRef), createdAt: new Date().toISOString() };
const res = await publishPrivate(ssbClient, content, [me, route.author]);
return { ...res, routeId: route.id, owner: route.author, booker: me };
},
async setBookingStatus(bookingId, status, { receipt = '' } = {}) {
const { ssbClient, priv } = await load();
const me = ssbClient.id;
const st = pick(BOOKING_STATUSES, status, '');
if (!st || st === 'PENDING') throw new Error('Invalid status');
const b = priv.bookings.find(x => x.id === bookingId);
if (!b) throw new Error('Booking not found');
const owner = safeText(b.c.owner);
if (OWNER_BOOKING_STATUSES.includes(st)) { if (me !== owner) throw new Error('Only the route owner can set this status'); }
else if (me !== b.author) throw new Error('Only the booker can cancel');
const content = { type: BOOKING_STATUS_TYPE, bookingId, routeId: String(b.c.routeId), status: st, ...(st === 'DELIVERED' ? { receipt: safeText(receipt) || `Delivered ${new Date().toISOString()}` } : {}), updatedAt: new Date().toISOString() };
const res = await publishPrivate(ssbClient, content, Array.from(new Set([me, b.author, owner].filter(Boolean))));
if (me === owner) {
try {
const fresh = await load();
const route = find(fresh.idx, b.c.routeId);
if (route && route.kind === 'TRIP' && route.seats) {
const taken = bookingsFor(fresh.priv, route.id, me).filter(x => x.status === 'CONFIRMED' || x.status === 'DELIVERED').reduce((sum, x) => sum + (x.seats || 1), 0);
if (taken !== route.seatsTaken) await publish(ssbClient, { ...routeContent(me, { seatsTaken: taken }, route), replaces: route.tipId });
}
} catch (_) {}
}
return { ...res, routeId: String(b.c.routeId), owner, booker: b.author, status: st };
},
async rate(id, { score, text = '' } = {}) {
const { ssbClient, idx, priv } = await load();
const me = ssbClient.id;
const route = find(idx, id);
if (!route) throw new Error('Route not found');
const view = decorate(route, priv, me);
if (!view.participated) throw new Error('Only participants can rate a route');
if (!view.closed) throw new Error('Routes are rated once they are closed');
if (view.myRating) throw new Error('You already rated this route');
const s = Math.min(5, Math.max(1, Math.round(num(score, 1) || 1)));
return publish(ssbClient, { type: RATING_TYPE, target: route.id, score: s, text: safeText(text), createdAt: new Date().toISOString() });
},
async getRouteById(id) {
const { ssbClient, idx, priv } = await load();
const route = find(idx, id);
return route ? decorate(route, priv, ssbClient.id) : null;
},
async myBookings() {
const { ssbClient, idx, priv } = await load();
const me = ssbClient.id;
const out = [];
for (const b of priv.bookings) {
if (b.author !== me) continue;
const route = find(idx, b.c.routeId);
if (!route) continue;
const view = decorate(route, priv, me);
const mine = view.bookings.find(x => x.id === b.id);
if (mine) out.push({ ...mine, route: view });
}
return out.sort((a, b) => b.ts - a.ts);
},
zonesOf(list) {
const counts = new Map();
for (const r of list || []) {
for (const z of [r.origin, r.destination]) {
const k = zoneKey(z);
if (!k) continue;
const prev = counts.get(k) || { label: safeText(z), count: 0 };
prev.count += 1;
counts.set(k, prev);
}
}
return Array.from(counts.values()).sort((a, b) => b.count - a.count || a.label.localeCompare(b.label)).slice(0, MAX_ZONES);
},
async listAll({ filter = 'all', q = '', zone = '' } = {}) {
const { ssbClient, idx, priv } = await load();
const me = ssbClient.id;
const f = String(filter || 'all').toLowerCase();
const needle = String(q || '').trim().toLowerCase();
const z = zoneKey(zone);
let out = Array.from(idx.routes.values()).map(r => decorate(r, priv, me));
if (f === 'mine') out = out.filter(r => r.isOwner);
else if (f === 'booked') out = out.filter(r => !!r.myBooking);
else if (f === 'recent') out = out.filter(r => r.lastActivityTs >= Date.now() - RECENT_MS);
else if (f === 'trips') out = out.filter(r => r.kind === 'TRIP');
else if (f === 'shipments') out = out.filter(r => r.kind === 'SHIPMENT');
else if (f === 'offers') out = out.filter(r => r.mode === 'OFFER');
else if (f === 'requests') out = out.filter(r => r.mode === 'REQUEST');
else if (f === 'upcoming') out = out.filter(r => r.status === 'OPEN');
else if (f === 'past') out = out.filter(r => r.status === 'PAST');
else if (f === 'closed') out = out.filter(r => r.status === 'CLOSED');
else if (f === 'history') out = out.filter(r => r.closed && r.participated);
else if (f === 'free') out = out.filter(r => r.priceType === 'FREE');
else if (f === 'eco') out = out.filter(r => r.priceType === 'ECO');
else if (f === 'time') out = out.filter(r => r.priceType === 'TIME');
else if (f === 'top') out = out.filter(r => r.ratingCount > 0);
if (z) out = out.filter(r => zoneKey(r.origin) === z || zoneKey(r.destination) === z);
if (needle) out = out.filter(r => [r.title, r.description, r.origin, r.destination, r.kind, r.mode, ...r.tags].some(v => String(v || '').toLowerCase().includes(needle)));
if (f === 'top') out.sort((a, b) => b.ratingAvg - a.ratingAvg || b.ratingCount - a.ratingCount);
else out.sort((a, b) => (a.closed === b.closed ? 0 : (a.closed ? 1 : -1)) || (a.dateTs && b.dateTs ? a.dateTs - b.dateTs : b.lastActivityTs - a.lastActivityTs));
return out;
},
async resolveRootId(id) {
const { idx } = await load();
const route = find(idx, id);
return route ? route.id : String(id || '');
}
};
};

151
src/models/logs_model.js
View file

@ -1,6 +1,6 @@
const pull = require('../server/node_modules/pull-stream');
const util = require('../server/node_modules/util');
const axios = require('../server/node_modules/axios');
const { requestScope } = require('./typed_log');
const util = require('util');
const { getConfig } = require('../configs/config-manager.js');
const { buildValidatedTombstoneSet } = require('./tombstone_validator');
@ -22,8 +22,8 @@ const FILTER_WINDOWS = {
const ACTION_TYPES = new Set([
'post', 'about', 'contact', 'feed', 'bookmark', 'image', 'audio', 'video',
'document', 'torrent', 'event', 'task', 'taskAssignment',
'votes', 'vote', 'report', 'tribe', 'chat', 'chatMessage', 'pad', 'padEntry',
'document', 'torrent', 'file', 'event', 'task', 'taskAssign',
'votes', 'vote', 'report', 'tribe', 'chat', 'chatMessage', 'pad', 'padEntry', 'room', 'roomMember',
'forum', 'market', 'job', 'project', 'industry', 'industryBuild', 'pixelia', 'map', 'mapMarker',
'shop', 'shopProduct', 'curriculum', 'gameScore',
'calendar', 'calendarDate', 'calendarNote',
@ -33,7 +33,29 @@ const ACTION_TYPES = new Set([
'courtsCase', 'courtsEvidence', 'courtsAnswer', 'courtsVerdict',
'courtsNomination', 'courtsNominationVote',
'courtsSettlementProposal', 'courtsSettlementAccepted',
'tribeParliamentCandidature', 'tribeParliamentRule'
'tribeParliamentCandidature', 'tribeParliamentRule',
'wikiPage', 'emergency', 'emergencyConfirm', 'emergencyUpdate', 'mailingList',
'logisticsRoute', 'logisticsRating', 'logisticsOpinion',
'podcast', 'podcastEpisode', 'podcastOpinion', 'podcastPlay',
'campaign', 'campaignSignature', 'campaignUpdate', 'campaignOpinion',
'audioOpinion', 'blogOpinion', 'bookmarkOpinion', 'documentOpinion', 'eventOpinion', 'feedOpinion',
'fileOpinion', 'imageOpinion', 'videoOpinion', 'torrentOpinion', 'housingOpinion', 'industryOpinion',
'marketOpinion', 'projectOpinion', 'reportOpinion', 'schoolOpinion', 'shopOpinion', 'taskOpinion',
'transferOpinion', 'votesOpinion', 'pollOpinion',
'calendarParticipant', 'chatMember', 'chatPin', 'chatReaction', 'padMember', 'mapMember',
'courtsJudge', 'courtsMediators', 'courtsSupport', 'courtsVisibility', 'courtsVoteLink',
'eventAttend', 'feed-action', 'spread', 'subscription', 'job_sub',
'housing', 'housingRequest',
'industryAllocation', 'industryApply', 'industryBlueprint', 'industryBuildStatus', 'industryBuildUpdate',
'industryContribution', 'industryMember', 'industryVote',
'larpHousePost', 'larpJoinHouse', 'larpLeaveLarp', 'larpTestAttempt',
'marketBid', 'marketPurchase', 'shopPurchase',
'parliamentCandidatureVote', 'tribeParliamentTerm',
'poll', 'pollVote', 'pollClose', 'votesVote',
'projectClaim', 'projectFollow', 'projectPledge', 'projectPledgeConfirm',
'reportConfirm', 'transferConfirm', 'ubiRefuse',
'schoolCourse', 'schoolEnroll', 'schoolLesson', 'schoolLessonMedia', 'schoolExam', 'schoolExamQuestion',
'schoolExamResult', 'schoolProgress', 'schoolCertificate', 'schoolCommentHide'
]);
const ACTION_PHRASES = {
@ -47,9 +69,10 @@ const ACTION_PHRASES = {
video: 'uploaded a video',
document: 'uploaded a document',
torrent: 'shared a torrent',
file: 'shared a file',
event: 'created an event',
task: 'created a task',
taskAssignment: 'updated a task assignment',
taskAssign: 'updated a task assignment',
votes: 'participated in a vote',
vote: 'cast a vote',
report: 'submitted a report',
@ -58,6 +81,8 @@ const ACTION_PHRASES = {
chatMessage: 'sent a chat message',
pad: 'worked on a collaborative pad',
padEntry: 'edited a pad entry',
room: 'opened a room',
roomMember: 'joined a room',
market: 'posted in the market',
forum: 'posted in the forum',
job: 'posted a job opportunity',
@ -91,7 +116,100 @@ const ACTION_PHRASES = {
courtsSettlementProposal: 'proposed a courts settlement',
courtsSettlementAccepted: 'accepted a courts settlement',
tribeParliamentCandidature: 'stood for a tribe parliament',
tribeParliamentRule: 'contributed a tribe parliament rule'
tribeParliamentRule: 'contributed a tribe parliament rule',
wikiPage: 'edited a wiki page',
emergency: 'reported an emergency',
emergencyConfirm: 'confirmed an emergency',
emergencyUpdate: 'posted an emergency update',
mailingList: 'created a mailing list',
logisticsRoute: 'published a logistics route',
logisticsRating: 'rated a logistics route',
logisticsOpinion: 'gave an opinion on a route',
podcast: 'created a podcast',
podcastEpisode: 'published a podcast episode',
podcastOpinion: 'gave an opinion on a podcast',
podcastPlay: 'listened to a podcast episode',
campaign: 'started a campaign',
campaignSignature: 'signed a campaign',
campaignUpdate: 'posted a campaign update',
campaignOpinion: 'gave an opinion on a campaign',
audioOpinion: 'gave an opinion on an audio',
blogOpinion: 'gave an opinion on a blog post',
bookmarkOpinion: 'gave an opinion on a bookmark',
documentOpinion: 'gave an opinion on a document',
eventOpinion: 'gave an opinion on an event',
feedOpinion: 'gave an opinion on a feed',
fileOpinion: 'gave an opinion on a file',
imageOpinion: 'gave an opinion on an image',
videoOpinion: 'gave an opinion on a video',
torrentOpinion: 'gave an opinion on a torrent',
housingOpinion: 'gave an opinion on a housing listing',
industryOpinion: 'gave an opinion on a facility',
marketOpinion: 'gave an opinion on a market item',
projectOpinion: 'gave an opinion on a project',
reportOpinion: 'gave an opinion on a report',
schoolOpinion: 'gave an opinion on a course',
shopOpinion: 'gave an opinion on a shop',
taskOpinion: 'gave an opinion on a task',
transferOpinion: 'gave an opinion on a transfer',
votesOpinion: 'gave an opinion on a votation',
pollOpinion: 'gave an opinion on a poll',
calendarParticipant: 'joined a calendar',
chatMember: 'joined a chat',
chatPin: 'pinned a chat message',
chatReaction: 'reacted to a chat message',
padMember: 'joined a pad',
mapMember: 'joined a map',
courtsJudge: 'chose a judge for a courts case',
courtsMediators: 'proposed mediators for a courts case',
courtsSupport: 'supported a courts case',
courtsVisibility: 'changed the visibility of a courts case',
courtsVoteLink: 'linked a vote to a courts case',
eventAttend: 'joined an event',
'feed-action': 'refeeded a feed',
spread: 'spread some content',
subscription: 'subscribed to some content',
job_sub: 'subscribed to a job',
housing: 'published a housing listing',
housingRequest: 'requested a housing',
industryAllocation: 'shared out the output of a production build',
industryApply: 'applied to join a facility',
industryBlueprint: 'published a blueprint',
industryBuildStatus: 'updated the status of a production build',
industryBuildUpdate: 'posted a production build update',
industryContribution: 'contributed to a production build',
industryMember: 'joined a facility',
industryVote: 'voted in a facility',
larpHousePost: 'posted in a L.A.R.P. house',
larpJoinHouse: 'joined a L.A.R.P. house',
larpLeaveLarp: 'left L.A.R.P.',
larpTestAttempt: 'took a L.A.R.P. test',
marketBid: 'placed a bid in the market',
marketPurchase: 'bought in the market',
shopPurchase: 'bought in a shop',
parliamentCandidatureVote: 'voted for a parliamentary candidature',
tribeParliamentTerm: 'took part in a tribe parliament term',
poll: 'created a poll',
pollVote: 'voted in a poll',
pollClose: 'closed a poll',
votesVote: 'voted in a votation',
projectClaim: 'claimed a project bounty',
projectFollow: 'followed a project',
projectPledge: 'pledged to a project',
projectPledgeConfirm: 'confirmed a project pledge',
reportConfirm: 'confirmed a report',
transferConfirm: 'confirmed a transfer',
ubiRefuse: 'declined the UBI',
schoolCourse: 'created a course',
schoolEnroll: 'enrolled in a course',
schoolLesson: 'published a lesson',
schoolLessonMedia: 'added media to a lesson',
schoolExam: 'created an exam',
schoolExamQuestion: 'added an exam question',
schoolExamResult: 'completed an exam',
schoolProgress: 'progressed in a course',
schoolCertificate: 'issued a course certificate',
schoolCommentHide: 'hid a comment in a course'
};
const compact = (s, n = 200) => String(s || '').replace(/\s+/g, ' ').trim().slice(0, n);
@ -111,7 +229,7 @@ module.exports = ({ cooler }) => {
const ssbClient = await openSsb();
const msgs = await new Promise((resolve, reject) =>
pull(
ssbClient.createLogStream({ reverse: true, limit: logLimit }),
ssbClient.createUserStream({ id: ssbClient.id, reverse: true, limit: logLimit }),
pull.collect((err, arr) => err ? reject(err) : resolve(arr))
)
);
@ -132,10 +250,11 @@ module.exports = ({ cooler }) => {
async function callAI(prompt) {
if (!prompt) return '';
try { if ((require('../configs/config-manager.js').getConfig().modules || {}).aiMod !== 'on') return ''; } catch (_) { return ''; }
const tryOnce = async () => {
try {
const res = await axios.post('http://localhost:4001/ai', { input: prompt, raw: true }, { timeout: 90000 });
return String(res?.data?.answer || '').trim();
const answer = await require('../AI/ai_client').ask({ system: 'You write one short first-person diary sentence. No IDs, hashes, quotes, lists or markdown.', input: prompt, maxTokens: 80 });
return String(answer || '').trim();
} catch { return ''; }
};
let out = await tryOnce();
@ -272,9 +391,10 @@ module.exports = ({ cooler }) => {
async function readAllLogMessages() {
const ssbClient = await openSsb();
const sealedOnly = ssbClient.private && typeof ssbClient.private.read === 'function';
const raw = await new Promise((resolve, reject) =>
pull(
ssbClient.createLogStream({ reverse: false, limit: logLimit }),
sealedOnly ? ssbClient.private.read({ reverse: true, limit: logLimit }) : ssbClient.createLogStream({ reverse: true, limit: logLimit }),
pull.collect((err, arr) => err ? reject(err) : resolve(arr))
)
);
@ -288,7 +408,7 @@ module.exports = ({ cooler }) => {
const tsIn = m.timestamp || valueIn?.timestamp || Date.now();
let dec;
try {
dec = ssbClient.private.unbox({ key: keyIn, value: valueIn, timestamp: tsIn });
dec = typeof valueIn.content === 'string' ? ssbClient.private.unbox({ key: keyIn, value: valueIn, timestamp: tsIn }) : (valueIn.private === true ? m : null);
} catch { continue; }
const v = dec?.value;
const c = v?.content;
@ -327,7 +447,9 @@ module.exports = ({ cooler }) => {
}
async function listLogs(filter = 'today') {
const items = await readAllLogMessages();
const store = requestScope.getStore();
if (store && !store.logsAll) store.logsAll = readAllLogMessages();
const items = await (store ? store.logsAll : readAllLogMessages());
const win = FILTER_WINDOWS[filter];
if (win === null || win === undefined) return items;
const cutoff = Date.now() - win;
@ -375,6 +497,7 @@ module.exports = ({ cooler }) => {
getLogById,
listLogs,
countLogs,
isAImodOn
isAImodOn,
FILTER_WINDOWS
};
};

437
src/models/mailing_model.js Normal file
View file

@ -0,0 +1,437 @@
const pull = require('../server/node_modules/pull-stream');
const crypto = require('crypto');
const { readTyped } = require('./typed_log');
const { buildValidatedTombstoneSet } = require('./tombstone_validator');
const longText = require('../backend/long_text');
const { getConfig } = require('../configs/config-manager.js');
const logLimit = getConfig().ssbLogStream?.limit || 1000;
const LIST_TYPE = 'mailingList';
const LEAVE_TYPE = 'mailingLeave';
const LIST_TYPES = ['OPEN', 'CLOSED'];
const STATUSES = ['ACTIVE', 'ARCHIVED'];
const BATCH = 6;
const RECENT_MS = 24 * 60 * 60 * 1000;
const safeText = (v) => String(v == null ? '' : v).trim();
const normU = (v) => String(v || '').trim().toUpperCase();
const normalizeListType = (v) => (LIST_TYPES.includes(normU(v)) ? normU(v) : 'OPEN');
const normalizeStatus = (v) => (STATUSES.includes(normU(v)) ? normU(v) : 'ACTIVE');
const normalizeTags = (raw) => {
if (!raw) return [];
const arr = Array.isArray(raw) ? raw : String(raw).split(',');
return Array.from(new Set(arr.map(t => String(t || '').trim()).filter(Boolean)));
};
const isFeedId = (v) => typeof v === 'string' && /^@[A-Za-z0-9+/=]+\.ed25519$/.test(v.trim());
const normalizeMembers = (raw) => {
if (!raw) return [];
const arr = Array.isArray(raw) ? raw : String(raw).split(/[\s,]+/);
return Array.from(new Set(arr.map(t => String(t || '').trim()).filter(isFeedId)));
};
const uniq = (list) => Array.from(new Set((list || []).filter(Boolean)));
const canonicalSubject = (s) => String(s || '').replace(/^\s*(RE:\s*)+/i, '').trim();
module.exports = ({ cooler, subscriptionsModel = null }) => {
let ssb;
const openSsb = async () => { if (!ssb) ssb = await cooler.open(); return ssb; };
const publish = (ssbClient, content) => new Promise((resolve, reject) => ssbClient.publish(content, (err, res) => err ? reject(err) : resolve(res)));
const publishPrivate = (ssbClient, content, recps) => new Promise((resolve, reject) => ssbClient.private.publish(content, recps, (err, res) => err ? reject(err) : resolve(res)));
const publishPrivateBatched = async (ssbClient, content, recipients) => {
const me = ssbClient.id;
const others = uniq(recipients).filter(id => id !== me);
const out = [];
if (!others.length) { out.push(await publishPrivate(ssbClient, content, [me])); return out; }
for (let i = 0; i < others.length; i += BATCH) {
out.push(await publishPrivate(ssbClient, content, uniq([me, ...others.slice(i, i + BATCH)])));
}
return out;
};
const privateState = new WeakMap();
const readPrivate = async (ssbClient) => {
const me = ssbClient.id;
let st = privateState.get(ssbClient);
if (!st || st.me !== me) {
st = { me, lastTs: 0, seen: new Set(), lists: [], leaves: [], posts: [], chunks: new Map(), tombClaims: new Map(), authorByKey: new Map() };
privateState.set(ssbClient, st);
}
const raw = await new Promise((resolve, reject) => {
pull(ssbClient.createLogStream(st.lastTs > 0 ? { reverse: false, gte: st.lastTs } : { reverse: false }), pull.collect((err, arr) => err ? reject(err) : resolve(arr)));
});
for (const m of raw) {
if (!m || !m.value) continue;
const recv = Number(m.timestamp) || 0;
if (recv > st.lastTs) st.lastTs = recv;
const content = m.value.content;
if (typeof content !== 'string') continue;
if (m.key) {
if (st.seen.has(m.key)) continue;
st.seen.add(m.key);
}
let dec;
try { dec = ssbClient.private.unbox({ key: m.key, value: m.value, timestamp: m.timestamp || m.value.timestamp }); } catch (_) { continue; }
const v = (dec && dec.value) || {};
const c = v.content;
const k = (dec && dec.key) || m.key;
if (!c || typeof c !== 'object' || !k) continue;
const ts = v.timestamp || m.timestamp || 0;
if (c.type === 'tombstone' && c.target) {
const set = st.tombClaims.get(c.target) || new Set();
set.add(v.author);
st.tombClaims.set(c.target, set);
continue;
}
if (c.type === longText.CHUNK_TYPE) {
const ch = longText.chunkOf(v.author, c);
if (ch) st.chunks.set(k, ch);
continue;
}
st.authorByKey.set(k, v.author);
if (c.type === LIST_TYPE) st.lists.push({ key: k, author: v.author, ts, seq: v.sequence || 0, arrival: st.lists.length, c });
else if (c.type === LEAVE_TYPE && c.list) st.leaves.push({ key: k, author: v.author, ts, list: String(c.list) });
else if (c.type === 'post' && c.list) {
const to = Array.isArray(c.to) ? c.to : [];
if (v.author === me || to.includes(me)) st.posts.push({ key: k, author: v.author, ts, c });
}
}
const tombed = new Set();
for (const [target, authors] of st.tombClaims.entries()) {
const orig = st.authorByKey.get(target);
for (const a of authors) if (a === orig || a === me) { tombed.add(target); break; }
}
return { lists: st.lists.slice(), leaves: st.leaves.slice(), posts: st.posts.filter(p => !tombed.has(p.key)), tombed, chunks: st.chunks };
};
const buildIndex = (publicMsgs, priv) => {
const tomb = buildValidatedTombstoneSet(publicMsgs);
const nodes = new Map();
const parentOf = new Map();
for (const m of publicMsgs) {
const v = m.value || {};
const c = v.content;
if (!c || typeof c !== 'object' || c.type !== LIST_TYPE) continue;
nodes.set(m.key, { key: m.key, author: v.author, ts: v.timestamp || m.timestamp || 0, seq: v.sequence || 0, arrival: Number(m.timestamp) || 0, c, closed: false });
if (typeof c.replaces === 'string') parentOf.set(m.key, c.replaces);
}
const rootOf = (key) => {
const seen = new Set();
let cur = key;
while (parentOf.has(cur) && !seen.has(cur)) { seen.add(cur); cur = parentOf.get(cur); }
return cur;
};
const versionsById = new Map();
for (const node of nodes.values()) {
const root = rootOf(node.key);
if (tomb.has(root)) continue;
if (!versionsById.has(root)) versionsById.set(root, []);
versionsById.get(root).push(node);
}
const privateById = new Map();
for (const n of priv.lists) {
const id = n.c.listId ? String(n.c.listId) : null;
if (!id || nodes.has(id)) continue;
if (!privateById.has(id)) privateById.set(id, []);
privateById.get(id).push({ ...n, closed: true });
}
for (const [id, versions] of privateById) versionsById.set(id, versions);
const ownerOf = (id, versions) => {
if (nodes.has(id)) return nodes.get(id).author;
const keys = new Set(versions.map(n => n.key));
const heads = versions.filter(n => !(typeof n.c.replaces === 'string' && keys.has(n.c.replaces)) && (typeof n.c.author !== 'string' || n.c.author === n.author));
heads.sort((a, b) => a.arrival - b.arrival);
return heads.length ? heads[0].author : null;
};
const lists = new Map();
for (const [id, versions] of versionsById) {
const owner = ownerOf(id, versions);
const own = versions.filter(v => v.author === owner).sort((a, b) => (a.seq - b.seq) || (a.ts - b.ts));
if (!own.length || own.some(v => v.closed && normU(v.c.status) === 'DELETED')) continue;
const first = own[0];
const tip = own[own.length - 1];
const c = tip.c;
const listType = first.closed ? 'CLOSED' : normalizeListType(c.listType);
const updatedAt = c.updatedAt || new Date(tip.ts).toISOString();
let members = first.closed ? uniq([owner, ...normalizeMembers(c.members)]) : [];
if (first.closed) {
const left = new Set(priv.leaves.filter(l => l.list === id && l.ts >= tip.ts).map(l => l.author));
members = members.filter(m => m === owner || !left.has(m));
}
lists.set(id, {
id, rootId: id, tipId: tip.key,
title: safeText(c.title), description: safeText(c.description),
listType, closed: listType === 'CLOSED', status: normalizeStatus(c.status), tags: normalizeTags(c.tags),
members, author: owner,
createdAt: first.c.createdAt || new Date(first.ts).toISOString(),
updatedAt, ts: tip.ts
});
}
return { lists, rootOf, nodes };
};
const historyFor = (priv, id, me) => {
const seen = new Map();
for (const p of priv.posts) {
if (String(p.c.list) !== String(id)) continue;
const mid = p.c.mid ? String(p.c.mid) : p.key;
if (seen.has(mid)) continue;
seen.set(mid, p);
}
const out = Array.from(seen.values()).map(p => ({
key: p.key, mid: p.c.mid ? String(p.c.mid) : p.key, list: String(p.c.list),
author: p.author, subject: safeText(p.c.subject), text: safeText(longText.joinText(p.c.text, p.c.chunks, longText.lookupIn(priv.chunks), p.author)),
thread: p.c.thread ? String(p.c.thread) : (p.c.mid ? String(p.c.mid) : p.key),
sentAt: p.c.sentAt || new Date(p.ts).toISOString(), ts: Date.parse(p.c.sentAt || '') || p.ts
}));
out.sort((a, b) => a.ts - b.ts);
return out;
};
const load = async () => {
const ssbClient = await openSsb();
const [publicMsgs, priv] = await Promise.all([readTyped(ssbClient, [LIST_TYPE, 'tombstone'], { limit: logLimit }), readPrivate(ssbClient)]);
const idx = buildIndex(publicMsgs, priv);
return { ssbClient, idx, priv };
};
const find = (idx, id) => {
const key = String(id || '');
if (idx.lists.has(key)) return idx.lists.get(key);
if (idx.nodes.has(key)) return idx.lists.get(idx.rootOf(key)) || null;
return null;
};
const subscribersMap = async (ids) => {
if (!subscriptionsModel || !ids.length) return new Map();
if (typeof subscriptionsModel.subscribersMap === 'function') {
try { return await subscriptionsModel.subscribersMap(ids); } catch (_) { return new Map(); }
}
const out = new Map();
for (const id of ids) {
try { out.set(id, await subscriptionsModel.listSubscribers(id)); } catch (_) { out.set(id, []); }
}
return out;
};
const decorate = async (ssbClient, list, priv, subs) => {
const me = ssbClient.id;
const history = historyFor(priv, list.id, me);
const subscribers = list.closed ? [] : ((subs && subs.get(list.id)) || []);
const participants = list.closed ? list.members : uniq([list.author, ...subscribers]);
const isMember = participants.includes(me);
const lastTs = history.length ? history[history.length - 1].ts : list.ts;
return {
...list,
subscribers, participants, participantCount: participants.length,
isMember, isOwner: list.author === me, canWrite: isMember && list.status === 'ACTIVE',
messageCount: history.length, threadCount: new Set(history.map(h => h.thread)).size,
lastActivityTs: lastTs, history
};
};
const visibleTo = (list, me) => !list.closed || list.members.includes(me);
const recipientsFor = async (ssbClient, list) => {
const me = ssbClient.id;
const subs = list.closed ? [] : ((await subscribersMap([list.id])).get(list.id) || []);
const all = list.closed ? list.members : uniq([list.author, ...subs]);
return { all, others: all.filter(id => id !== me) };
};
const publishListVersion = async (ssbClient, content, list) => {
if (!list || !list.closed) return publish(ssbClient, content);
const res = await publishPrivateBatched(ssbClient, content, list.members);
return res[0];
};
return {
LIST_TYPE,
LIST_TYPES,
STATUSES,
async createList({ title, description, listType, members, tags }) {
const ssbClient = await openSsb();
const me = ssbClient.id;
const now = new Date().toISOString();
const t = safeText(title);
if (!t) throw new Error('Title is required');
const type = normalizeListType(listType);
const content = {
type: LIST_TYPE,
title: t,
description: safeText(description),
listType: type,
status: 'ACTIVE',
tags: normalizeTags(tags),
author: me,
createdAt: now,
updatedAt: now
};
if (type === 'CLOSED') {
const listId = crypto.randomBytes(16).toString('hex');
const mem = uniq([me, ...normalizeMembers(members)]);
const res = await publishPrivateBatched(ssbClient, { ...content, listId, members: mem, private: true }, mem);
return { key: listId, rootId: listId, msgKey: res[0].key };
}
const res = await publish(ssbClient, content);
return { key: res.key, rootId: res.key, msgKey: res.key };
},
async updateList(id, data = {}) {
const { ssbClient, idx } = await load();
const me = ssbClient.id;
const list = find(idx, id);
if (!list) throw new Error('Mailing list not found');
if (list.author !== me) throw new Error('Only the author can update this mailing list');
const now = new Date().toISOString();
const nextMembers = list.closed && data.members !== undefined ? uniq([me, ...normalizeMembers(data.members)]) : list.members;
const content = {
type: LIST_TYPE,
title: data.title !== undefined && safeText(data.title) ? safeText(data.title) : list.title,
description: data.description !== undefined ? safeText(data.description) : list.description,
listType: list.listType,
status: data.status !== undefined ? normalizeStatus(data.status) : list.status,
tags: data.tags !== undefined ? normalizeTags(data.tags) : list.tags,
author: me,
createdAt: list.createdAt,
updatedAt: now,
replaces: list.tipId
};
if (list.closed) {
const res = await publishPrivateBatched(ssbClient, { ...content, listId: list.id, members: nextMembers, private: true }, uniq([...list.members, ...nextMembers]));
return { key: res[0].key, rootId: list.id };
}
const res = await publish(ssbClient, content);
return { key: res.key, rootId: list.id };
},
async setStatus(id, status) {
return this.updateList(id, { status: normalizeStatus(status) });
},
async deleteList(id) {
const { ssbClient, idx } = await load();
const me = ssbClient.id;
const list = find(idx, id);
if (!list) throw new Error('Mailing list not found');
if (list.author !== me) throw new Error('Only the author can delete this mailing list');
if (list.closed) {
await publishPrivateBatched(ssbClient, { type: LIST_TYPE, listId: list.id, title: list.title, status: 'DELETED', author: me, createdAt: list.createdAt, updatedAt: new Date().toISOString(), replaces: list.tipId, private: true }, list.members);
return { key: list.id };
}
await publish(ssbClient, { type: 'tombstone', target: list.id, deletedAt: new Date().toISOString(), author: me });
return { key: list.id };
},
async leaveList(id) {
const { ssbClient, idx } = await load();
const me = ssbClient.id;
const list = find(idx, id);
if (!list) throw new Error('Mailing list not found');
if (!list.closed) {
if (!subscriptionsModel) throw new Error('Subscriptions unavailable');
return subscriptionsModel.setSubscription(list.id, 'mailing', false);
}
if (list.author === me) throw new Error('The author cannot leave the mailing list');
if (!list.members.includes(me)) throw new Error('Not a member');
return publishPrivateBatched(ssbClient, { type: LEAVE_TYPE, list: list.id, private: true, createdAt: new Date().toISOString() }, [list.author]);
},
async sendMessage(id, { subject, text, thread = '', replyTo = '' } = {}) {
const { ssbClient, idx, priv } = await load();
const me = ssbClient.id;
const list = find(idx, id);
if (!list || !visibleTo(list, me)) throw new Error('Mailing list not found');
if (list.status !== 'ACTIVE') throw new Error('This mailing list is archived');
let { all, others } = await recipientsFor(ssbClient, list);
if (!all.includes(me) && !list.closed && subscriptionsModel) {
await subscriptionsModel.setSubscription(list.id, 'mailing', true);
({ all, others } = await recipientsFor(ssbClient, list));
if (!all.includes(me)) { all = uniq([...all, me]); others = all.filter(id => id !== me); }
}
if (!all.includes(me)) throw new Error('Only participants can write to this mailing list');
const body = safeText(text);
if (!body) throw new Error('Text is required');
const mid = crypto.randomBytes(12).toString('hex');
let threadId = safeText(thread);
let subj = safeText(subject);
if (!threadId && replyTo) {
const parent = historyFor(priv, list.id, me).find(h => h.key === replyTo || h.mid === replyTo);
if (parent) { threadId = parent.thread; if (!subj) subj = `RE: ${canonicalSubject(parent.subject)}`; }
}
const content = {
type: 'post',
from: me,
to: [],
subject: subj || list.title,
text: body,
sentAt: new Date().toISOString(),
private: true,
list: list.id,
mid,
thread: threadId || mid
};
const sendBatch = async (recps) => {
const sealed = await longText.chunkContent({ ...content, to: recps }, 'text', { maxBytes: longText.CONTENT_BYTES.boxed, extra: { private: true }, publish: (c) => publishPrivate(ssbClient, c, recps) });
return publishPrivate(ssbClient, sealed, recps);
};
const out = [];
if (!others.length) out.push(await sendBatch([me]));
for (let i = 0; i < others.length; i += BATCH) {
out.push(await sendBatch(uniq([me, ...others.slice(i, i + BATCH)])));
}
return { key: out[0].key, mid, thread: content.thread, batches: out.length };
},
async getListById(id) {
const { ssbClient, idx, priv } = await load();
const list = find(idx, id);
if (!list || !visibleTo(list, ssbClient.id)) return null;
return decorate(ssbClient, list, priv, list.closed ? null : await subscribersMap([list.id]));
},
async listAll({ filter = 'all', q = '' } = {}) {
const { ssbClient, idx, priv } = await load();
const me = ssbClient.id;
const f = String(filter || 'all').toLowerCase();
const needle = String(q || '').trim().toLowerCase();
const visible = Array.from(idx.lists.values()).filter(l => visibleTo(l, me));
const subs = await subscribersMap(visible.filter(l => !l.closed).map(l => l.id));
let out = [];
for (const list of visible) out.push(await decorate(ssbClient, list, priv, subs));
if (f === 'mine') out = out.filter(l => l.author === me);
else if (f === 'subscribed') out = out.filter(l => l.isMember && l.author !== me);
else if (f === 'recent') out = out.filter(l => l.lastActivityTs >= Date.now() - RECENT_MS);
else if (f === 'open') out = out.filter(l => !l.closed);
else if (f === 'closed') out = out.filter(l => l.closed);
else if (f === 'active') out = out.filter(l => l.status === 'ACTIVE');
else if (f === 'archived') out = out.filter(l => l.status === 'ARCHIVED');
if (needle) out = out.filter(l => [l.title, l.description, ...l.tags].some(v => String(v || '').toLowerCase().includes(needle)));
out.sort((a, b) => b.lastActivityTs - a.lastActivityTs);
return out;
},
async composerEntries() {
const all = await this.listAll({ filter: 'all' });
return all.filter(l => l.isMember && l.status === 'ACTIVE').map(l => ({ target: l.id, scope: 'mailing', title: l.title, owner: l.author, count: l.participantCount }));
},
async titlesFor(ids) {
const out = new Map();
const wanted = new Set((ids || []).map(String));
if (!wanted.size) return out;
const { ssbClient, idx } = await load();
for (const list of idx.lists.values()) {
if (wanted.has(list.id) && visibleTo(list, ssbClient.id)) out.set(list.id, { title: list.title, status: list.status, closed: list.closed });
}
return out;
},
async resolveRootId(id) {
const { idx } = await load();
const list = find(idx, id);
return list ? list.id : String(id || '');
}
};
};

814
src/models/main_models.js
View file

File diff suppressed because it is too large Load diff

226
src/models/maps_model.js
View file

@ -3,6 +3,7 @@ const crypto = require("crypto");
const { getConfig } = require("../configs/config-manager.js");
const { buildValidatedTombstoneSet } = require('./tombstone_validator');
const { collabContent, openInviteOf } = require('../backend/collab_content');
const { readTyped, memoIndex } = require("./typed_log");
const mapCollab = collabContent({ membersField: 'members', undecField: 'encrypted', contentFields: ['title', 'description', 'image'], listFields: ['tags', 'invites', 'markers'] });
const logLimit = getConfig().ssbLogStream?.limit || 1000;
@ -34,6 +35,16 @@ module.exports = ({ cooler, tribeCrypto, mapCrypto, tribesModel }) => {
return (tribeCrypto && tribeCrypto.getKeys(rid)) || [];
};
const lookupGen = (rid) => ((ownCrypto && ownCrypto.getGen(rid)) || (tribeCrypto && tribeCrypto.getGen(rid)) || 0);
const keyProofFor = (keyHex, id) => { try { return crypto.createHmac("sha256", Buffer.from(keyHex, "hex")).update(String(id), "utf8").digest("hex"); } catch (_) { return null; } };
const hashCode = (code, salt) => (tribeCrypto ? tribeCrypto.hashInviteCode(code, salt) : null);
const inviteMatches = (inv, code) => {
if (!code) return false;
if (typeof inv === "string") return inv === code;
if (!inv || typeof inv !== "object") return false;
if (typeof inv.code === "string") return inv.code === code;
return typeof inv.ch === "string" && hashCode(code, inv.salt) === inv.ch;
};
const inviteToken = (inv) => (inv && typeof inv === "object" && typeof inv.ch === "string") ? inv.ch : (typeof inv === "string" ? inv : (inv && inv.code) || "");
const rotateMapKey = async (rootId, remainingMembers) => {
if (!ownCrypto || !tribeCrypto || !rootId) return;
@ -41,11 +52,10 @@ module.exports = ({ cooler, tribeCrypto, mapCrypto, tribesModel }) => {
if (!existing) return;
const newKey = ownCrypto.generateTribeKey();
const newGen = ownCrypto.addNewKey(rootId, newKey);
if (!Array.isArray(remainingMembers) || !remainingMembers.length) return;
const ssbClient = await openSsb();
const ssbKeys = require("../server/node_modules/ssb-keys");
const memberKeys = {};
for (const m of remainingMembers) {
for (const m of new Set([ssbClient.id, ...(Array.isArray(remainingMembers) ? remainingMembers : [])])) {
try { memberKeys[m] = tribeCrypto.boxKeyForMember(newKey, m, ssbKeys); } catch (_) {}
}
if (Object.keys(memberKeys).length) {
@ -61,10 +71,13 @@ module.exports = ({ cooler, tribeCrypto, mapCrypto, tribesModel }) => {
const ssbClient = await openSsb();
const ssbKeys = require("../server/node_modules/ssb-keys");
const config = require("../server/ssb_config");
const msgs = await new Promise((resolve, reject) => pull(ssbClient.createLogStream({ limit: logLimit }), pull.collect((e, m) => e ? reject(e) : resolve(m))));
const msgs = await getAllMessages(ssbClient);
const idx = buildIndex(unwrapForIndex(msgs));
const trusted = (root, author) => { const n = idx.nodes.get(root); return !!author && !!n && (author === n.author || author === ssbClient.id); };
for (const m of msgs) {
const c = m.value && m.value.content;
if (!c || c.type !== "tribe-keys") continue;
if (!trusted(c.tribeId, m.value.author)) continue;
const memberKeys = c.memberKeys;
if (!memberKeys || typeof memberKeys !== "object") continue;
const boxed = memberKeys[ssbClient.id];
@ -148,13 +161,9 @@ module.exports = ({ cooler, tribeCrypto, mapCrypto, tribesModel }) => {
}
};
const getAllMessages = async (ssbClient) =>
new Promise((resolve, reject) => {
pull(
ssbClient.createLogStream({ limit: logLimit }),
pull.collect((err, msgs) => (err ? reject(err) : resolve(msgs)))
);
});
const MAP_TYPES = ["map", "mapMarker", "mapMember", "tribe-msg", "tribe-keys", "tombstone"];
const getAllMessages = async (ssbClient) => readTyped(ssbClient, MAP_TYPES, { limit: logLimit });
const getMsg = async (ssbClient, key) =>
new Promise((resolve) => {
@ -185,7 +194,7 @@ module.exports = ({ cooler, tribeCrypto, mapCrypto, tribesModel }) => {
}
if (c.type === "mapMember" && c.target) {
memberMsgs.push({ target: c.target, member: c.member, on: c.on !== false, author: v.author, ts: v.timestamp || m.timestamp || 0, code: typeof c.code === "string" ? c.code : "" });
memberMsgs.push({ target: c.target, member: c.member, on: c.on !== false, author: v.author, ts: v.timestamp || m.timestamp || 0, code: typeof c.code === "string" ? c.code : "", keyProof: typeof c.keyProof === "string" ? c.keyProof : "" });
continue;
}
@ -207,7 +216,7 @@ module.exports = ({ cooler, tribeCrypto, mapCrypto, tribesModel }) => {
if (c.type !== "map") continue;
const ts = v.timestamp || m.timestamp || 0;
nodes.set(k, { key: k, ts, c, author: v.author });
nodes.set(k, { key: k, ts, seq: v.sequence || 0, c, author: v.author });
authorByKey.set(k, v.author);
if (c.replaces) {
@ -235,21 +244,15 @@ module.exports = ({ cooler, tribeCrypto, mapCrypto, tribesModel }) => {
return cur;
};
const contentTipOf = (root) => {
const rn = nodes.get(root);
if (!rn) return root;
let cur = root, best = root, g = 0;
const seen = new Set();
while (child.has(cur) && !seen.has(cur) && g++ < 100000) {
seen.add(cur);
const next = child.get(cur);
const n = nodes.get(next);
if (!n) break;
if (n.author === rn.author && !tomb.has(next)) best = next;
cur = next;
}
return best;
};
const ownerTip = new Map();
for (const [k, n] of nodes) {
const r = rootOf(k);
const rn = nodes.get(r);
if (!rn || n.author !== rn.author) continue;
const prev = ownerTip.has(r) ? nodes.get(ownerTip.get(r)) : null;
if (!prev || n.seq > prev.seq || (n.seq === prev.seq && n.ts >= prev.ts)) ownerTip.set(r, k);
}
const contentTipOf = (root) => ownerTip.get(root) || root;
for (const t of tombRequests) {
const targetAuthor = authorByKey.get(t.target);
@ -268,8 +271,21 @@ module.exports = ({ cooler, tribeCrypto, mapCrypto, tribesModel }) => {
const forward = new Map();
for (const [newId, oldId] of parent.entries()) forward.set(oldId, newId);
const invitesOf = (root) => {
const out = [];
for (const id of [root, contentTipOf(root)]) { const n = nodes.get(id); if (n && Array.isArray(n.c.invites)) out.push(...n.c.invites); }
return out;
};
const selfJoinAllowed = (root, mm) => {
const oc = (nodes.get(root) || {}).c || {};
if (!oc.encryptedPayload || oc.tribeId) return true;
if (invitesOf(root).some(inv => inv && typeof inv === "object" && inv.public === true)) return true;
return !!mm.keyProof && lookupKeys(root).some(k => keyProofFor(k, mm.member) === mm.keyProof);
};
const memberByRoot = new Map();
const consumedByRoot = new Map();
const codeClaimsByRoot = new Map();
for (const mm of memberMsgs) {
if (!nodes.has(mm.target)) continue;
const r = rootOf(mm.target);
@ -279,6 +295,11 @@ module.exports = ({ cooler, tribeCrypto, mapCrypto, tribesModel }) => {
const byOwner = mm.author === ownerAuthor;
if (!self && !byOwner) continue;
if (!mm.member) continue;
if (mm.on && mm.code) {
if (!codeClaimsByRoot.has(r)) codeClaimsByRoot.set(r, []);
codeClaimsByRoot.get(r).push({ code: mm.code, member: mm.member, keyProof: mm.keyProof, byOwner });
}
if (self && !byOwner && mm.on && !selfJoinAllowed(r, mm)) continue;
if (mm.on && mm.code) {
if (!consumedByRoot.has(r)) consumedByRoot.set(r, new Set());
consumedByRoot.get(r).add(mm.code);
@ -289,12 +310,24 @@ module.exports = ({ cooler, tribeCrypto, mapCrypto, tribesModel }) => {
if (!prev || mm.ts >= prev.ts) m2.set(mm.member, { on: mm.on, ts: mm.ts });
}
const codeClaims = (root, codes) => (codeClaimsByRoot.get(root) || []).filter(c => codes.includes(c.code));
const isCodeConsumed = (root, code) => {
if (!code) return false;
const set = consumedByRoot.get(root);
return !!(set && set.has(code));
};
for (const [k, n] of nodes) {
const r = rootOf(k);
const rn = nodes.get(r);
if (!rn || n.author === rn.author || !Array.isArray(n.c.members) || !selfJoinAllowed(r, { member: n.author })) continue;
if (!memberByRoot.has(r)) memberByRoot.set(r, new Map());
const m2 = memberByRoot.get(r);
const prev = m2.get(n.author);
if (!prev || n.ts >= prev.ts) m2.set(n.author, { on: n.c.members.includes(n.author), ts: n.ts });
}
const resolveMembers = (root) => {
const ownerNode = nodes.get(root);
const oc = ownerNode ? ownerNode.c : {};
@ -305,29 +338,31 @@ module.exports = ({ cooler, tribeCrypto, mapCrypto, tribesModel }) => {
return [...set];
};
const ensureMemberKeys = async (ssbClient, messages, items) => {
if (!tribeCrypto) return;
const distributed = new Map();
return { tomb, nodes, parent, child, strictChild, rootOf, tipOf, contentTipOf, tipByRoot, contentTipByRoot, forward, markers, rawMarkers, resolveMembers, isCodeConsumed, codeClaims };
};
const ensureMemberKeys = async (ssbClient, messages, items) => {
if (!tribeCrypto || !ownCrypto) return;
const leavesByRoot = new Map();
for (const m of messages) {
const c = m.value && m.value.content;
if (!c || c.type !== "tribe-keys" || !c.tribeId) continue;
const mk = c.memberKeys;
if (!mk || typeof mk !== "object") continue;
if (!distributed.has(c.tribeId)) distributed.set(c.tribeId, new Set());
for (const id of Object.keys(mk)) distributed.get(c.tribeId).add(id);
if (!c || c.type !== "mapMember" || c.on !== false || !c.target || !c.member) continue;
if (!leavesByRoot.has(c.target)) leavesByRoot.set(c.target, []);
leavesByRoot.get(c.target).push({ member: c.member, author: m.value.author, ts: Number(m.timestamp || m.value.timestamp || 0) });
}
const ssbKeys = require("../server/node_modules/ssb-keys");
for (const item of (Array.isArray(items) ? items : [])) {
if (!item || item.encrypted) continue;
if (!item || item.encrypted || item.author !== ssbClient.id) continue;
const rootId = item.rootId;
if (!rootId) continue;
if (!rootId || !lookupKey(rootId)) continue;
const members = Array.isArray(item.members) ? item.members : [];
const leaves = (leavesByRoot.get(rootId) || []).filter(l => l.author === l.member || l.author === item.author);
const plan = ownCrypto.keyPlan({ rootId, ownerId: item.author, gen: lookupGen(rootId), members, messages, leaves });
if (plan.rotate) { await rotateMapKey(rootId, members); continue; }
if (!plan.missing.length) continue;
const key = lookupKey(rootId);
if (!key) continue;
const have = distributed.get(rootId) || new Set();
const missing = (Array.isArray(item.members) ? item.members : []).filter(m => m && m !== ssbClient.id && !have.has(m));
if (!missing.length) continue;
const memberKeys = {};
for (const m of missing) {
for (const m of plan.missing) {
try { memberKeys[m] = tribeCrypto.boxKeyForMember(key, m, ssbKeys) } catch (_) {}
}
if (!Object.keys(memberKeys).length) continue;
@ -337,13 +372,11 @@ module.exports = ({ cooler, tribeCrypto, mapCrypto, tribesModel }) => {
}
}
return { tomb, nodes, parent, child, strictChild, rootOf, tipOf, contentTipOf, tipByRoot, contentTipByRoot, forward, markers, rawMarkers, resolveMembers, isCodeConsumed };
};
const expandMarkers = async (idx) => {
for (const [mapId, raws] of idx.rawMarkers.entries()) {
const list = [];
for (const r of raws) {
if (idx.tomb.has(r.key)) continue;
let c = r.c;
if (c.encryptedPayload && tribeCrypto && tribesModel) {
const dec = await tribeCrypto.decryptFromTribe(c, tribesModel);
@ -364,6 +397,15 @@ module.exports = ({ cooler, tribeCrypto, mapCrypto, tribesModel }) => {
}
};
const markersFor = (idx, tipId, rootId) => {
const seen = new Set();
return safeArr(idx.markers.get(tipId)).concat(safeArr(idx.markers.get(rootId))).filter((mk) => {
if (seen.has(mk.key)) return false;
seen.add(mk.key);
return true;
});
};
const buildMap = (node, rootId, viewerId, markerList = [], members = null) => {
const c = node.c || {};
@ -384,6 +426,7 @@ module.exports = ({ cooler, tribeCrypto, mapCrypto, tribesModel }) => {
invites: Array.isArray(c.invites) ? c.invites : [],
tribeId: c.tribeId || null,
encrypted: !!undec,
contentEncrypted: !!(c.encryptedPayload || c._decrypted === true),
createdAt: c.createdAt || new Date(node.ts).toISOString(),
updatedAt: c.updatedAt || null,
markers: markerList.filter((mk) => !mk.tombstoned)
@ -396,7 +439,7 @@ module.exports = ({ cooler, tribeCrypto, mapCrypto, tribesModel }) => {
if (idx.tomb.has(tipId)) continue;
const node = idx.nodes.get(tipId);
if (!node) continue;
const markerList = safeArr(idx.markers.get(tipId)).concat(safeArr(idx.markers.get(rootId)));
const markerList = markersFor(idx, tipId, rootId);
items.push(buildMap(node, rootId, viewerId, markerList, idx.resolveMembers(rootId)));
}
return items;
@ -411,7 +454,7 @@ module.exports = ({ cooler, tribeCrypto, mapCrypto, tribesModel }) => {
if (!ownCrypto || typeof ownCrypto.getAllRootIds !== "function") return 0;
try {
const ssbClient = await openSsb();
const messages = await new Promise((resolve, reject) => pull(ssbClient.createLogStream({ limit: logLimit }), pull.collect((e, m) => e ? reject(e) : resolve(m))));
const messages = await getAllMessages(ssbClient);
const tomb = buildValidatedTombstoneSet(messages);
const all = ownCrypto.getAllRootIds();
let removed = 0;
@ -431,11 +474,9 @@ module.exports = ({ cooler, tribeCrypto, mapCrypto, tribesModel }) => {
if (!map) throw new Error("Map not found");
if (map.author === userId) throw new Error("Author cannot leave their own map");
if (!Array.isArray(map.members) || !map.members.includes(userId)) return;
const members = map.members.filter(m => m !== userId);
const rootId = await this.resolveRootId(map.rootId || map.key);
const content = { type: "mapMember", target: rootId, member: userId, on: false, createdAt: new Date().toISOString() };
const result = await new Promise((resolve, reject) => ssbClient.publish(content, (e, r) => e ? reject(e) : resolve(r)));
try { await rotateMapKey(rootId, members); } catch (_) {}
return result;
},
@ -587,7 +628,7 @@ module.exports = ({ cooler, tribeCrypto, mapCrypto, tribesModel }) => {
if (effectiveTribeId) {
updated = await encryptIfTribe(updated);
} else if (mType !== "SINGLE") {
} else if (mType !== "OPEN") {
updated = encryptStandalone(updated, rootId);
}
@ -623,9 +664,10 @@ module.exports = ({ cooler, tribeCrypto, mapCrypto, tribesModel }) => {
const messages = await getAllMessages(ssbClient);
const idx = buildIndex(unwrapForIndex(messages));
await decryptIndexNodes(idx);
let tipId = mapId;
while (idx.forward.has(tipId)) tipId = idx.forward.get(tipId);
const rootId = idx.rootOf(mapId);
const tipId = idx.contentTipOf(rootId);
if (idx.tomb.has(tipId)) throw new Error("Map not found");
const node = idx.nodes.get(tipId);
@ -634,12 +676,11 @@ module.exports = ({ cooler, tribeCrypto, mapCrypto, tribesModel }) => {
const mapDecrypted = await decryptIfTribe(node.c);
assertReadable(mapDecrypted, "Map");
const mapType = mapDecrypted.mapType || node.c.mapType || "SINGLE";
const mapAuthor = mapDecrypted.author || node.c.author;
const mapAuthor = node.author;
if (mapType === "SINGLE") throw new Error("Map does not allow markers");
if (mapType === "CLOSED" && mapAuthor !== userId) throw new Error("Only the map creator can add markers");
const now = new Date().toISOString();
const rootId = idx.rootOf(tipId);
let content = {
type: "mapMarker",
mapId: rootId,
@ -664,6 +705,24 @@ module.exports = ({ cooler, tribeCrypto, mapCrypto, tribesModel }) => {
});
},
async deleteMarker(markerId) {
const ssbClient = await openSsb();
const userId = ssbClient.id;
const msg = await getMsg(ssbClient, markerId);
if (!msg || !msg.content) throw new Error("Marker not found");
let c = msg.content;
if (c.encryptedPayload && tribeCrypto && tribesModel) {
const dec = await tribeCrypto.decryptFromTribe(c, tribesModel);
if (dec && !dec._undecryptable) c = dec;
}
if (c.type !== "mapMarker" && !c.encryptedPayload) throw new Error("Marker not found");
if (msg.author !== userId) throw new Error("Not the author");
const tombstone = await tombFor(markerId, c.tribeId, userId);
return new Promise((resolve, reject) => {
ssbClient.publish(tombstone, (err, res) => (err ? reject(err) : resolve(res)));
});
},
async listAll(filterOrOpts = "all", maybeOpts = {}) {
const ssbClient = await openSsb();
@ -703,9 +762,7 @@ module.exports = ({ cooler, tribeCrypto, mapCrypto, tribesModel }) => {
const viewer = viewerId || ssbClient.id;
const messages = await getAllMessages(ssbClient);
const idx = buildIndex(unwrapForIndex(messages));
await decryptIndexNodes(idx);
await expandMarkers(idx);
const idx = await memoIndex('maps', messages, async (m) => { const built = buildIndex(unwrapForIndex(m)); await decryptIndexNodes(built); await expandMarkers(built); return built; });
let root = id;
while (idx.parent.has(root)) root = idx.parent.get(root);
@ -721,11 +778,11 @@ module.exports = ({ cooler, tribeCrypto, mapCrypto, tribesModel }) => {
const dec = await tribeCrypto.decryptFromTribe(c, tribesModel);
c = dec && !dec._undecryptable ? { ...dec, _decrypted: true } : { ...c, _decrypted: false };
}
const markerList = safeArr(idx.markers.get(tip)).concat(safeArr(idx.markers.get(root)));
const markerList = markersFor(idx, tip, root);
return buildMap({ key: tip, ts: msg.timestamp || 0, c, author: msg.author }, root, viewer, markerList, idx.resolveMembers(root));
}
const markerList = safeArr(idx.markers.get(tip)).concat(safeArr(idx.markers.get(root)));
const markerList = markersFor(idx, tip, root);
const map = buildMap(node, root, viewer, markerList, idx.resolveMembers(root));
return mapCollab.fold(map, collectMaps(idx, viewer));
},
@ -740,8 +797,11 @@ module.exports = ({ cooler, tribeCrypto, mapCrypto, tribesModel }) => {
let invite = code;
if (tribeCrypto && !map.tribeId) {
const inviteSalt = tribeCrypto.generateInviteSalt();
const ident = opts.public ? { code, public: true } : { ch: hashCode(code, inviteSalt) };
const ekChain = tribeCrypto.encryptChainForInvite([map.rootId || map.key], code, inviteSalt);
if (ekChain) invite = { code, ekChain, salt: inviteSalt, gen: lookupGen(map.rootId || map.key) || 1, ...(opts.public ? { public: true } : {}) };
const mapKey = ekChain ? null : lookupKey(map.rootId || map.key);
if (ekChain) invite = { ...ident, ekChain, salt: inviteSalt, gen: lookupGen(map.rootId || map.key) || 1 };
else if (mapKey) invite = { ...ident, ek: tribeCrypto.encryptForInvite(mapKey, code, inviteSalt), salt: inviteSalt, gen: lookupGen(map.rootId || map.key) || 1 };
}
if (opts.public && typeof invite !== "object") invite = { code, public: true };
const tipId = await this.resolveCurrentId(mapId);
@ -834,6 +894,7 @@ module.exports = ({ cooler, tribeCrypto, mapCrypto, tribesModel }) => {
await new Promise((resolve, reject) => ssbClient.publish(updated, (e, r) => e ? reject(e) : resolve(r)));
const tomb1 = await tombFor(tipId, effectiveTribeId, userId);
await new Promise((resolve, reject) => ssbClient.publish(tomb1, e => e ? reject(e) : resolve()));
if (!effectiveTribeId) await rotateMapKey(rootId, Array.isArray(map.members) ? map.members : []);
},
async joinByInvite(code) {
@ -848,8 +909,7 @@ module.exports = ({ cooler, tribeCrypto, mapCrypto, tribesModel }) => {
for (const m of maps) {
const invs = Array.isArray(m.invites) ? m.invites : [];
for (const inv of invs) {
if (typeof inv === "string" && inv === code) { matched = m; matchedInvite = inv; break; }
if (typeof inv === "object" && inv.code === code) { matched = m; matchedInvite = inv; break; }
if (inviteMatches(inv, code)) { matched = m; matchedInvite = inv; break; }
}
if (matched) break;
}
@ -857,30 +917,38 @@ module.exports = ({ cooler, tribeCrypto, mapCrypto, tribesModel }) => {
if (Array.isArray(matched.members) && matched.members.includes(userId)) throw new Error("Already a member");
const isPublic = (typeof matchedInvite === "object" && matchedInvite.public === true);
const rootId = await this.resolveRootId(matched.rootId || matched.key);
if (!isPublic) {
const idx = buildIndex(unwrapForIndex(await getAllMessages(ssbClient)));
if (idx.isCodeConsumed(rootId, code)) throw new Error("Invite already used");
}
const token = inviteToken(matchedInvite) || code;
let mapKey = null;
let chain = null;
if (tribeCrypto && typeof matchedInvite === "object") {
if (matchedInvite.ekChain) {
const chain = tribeCrypto.decryptChainFromInvite(matchedInvite.ekChain, code, matchedInvite.salt, 3);
if (Array.isArray(chain) && chain.length) {
for (const entry of chain) {
if (Array.isArray(entry.keys) && entry.keys.length) {
tribeCrypto.setKeys(entry.rootId, entry.keys, entry.gen || entry.keys.length);
} else if (entry.key) {
tribeCrypto.setKey(entry.rootId, entry.key, entry.gen || 1);
}
}
mapKey = chain[0].key;
}
chain = tribeCrypto.decryptChainFromInvite(matchedInvite.ekChain, code, matchedInvite.salt, 3);
if (Array.isArray(chain) && chain.length) mapKey = chain[0].key || (Array.isArray(chain[0].keys) ? chain[0].keys[0] : null);
else chain = null;
} else if (matchedInvite.ek) {
mapKey = tribeCrypto.decryptFromInvite(matchedInvite.ek, code, matchedInvite.salt);
ownCrypto.setKey(matched.rootId || matched.key, mapKey, matchedInvite.gen || 1);
}
}
await new Promise((resolve, reject) => ssbClient.publish({ type: "mapMember", target: rootId, member: userId, on: true, createdAt: new Date().toISOString(), ...(isPublic ? {} : { code }) }, (e, r) => e ? reject(e) : resolve(r)));
if (!isPublic) {
const idx = buildIndex(unwrapForIndex(await getAllMessages(ssbClient)));
if (idx.isCodeConsumed(rootId, token) || idx.isCodeConsumed(rootId, code)) throw new Error("Invite already used");
const claims = idx.codeClaims(rootId, [token, code]).filter(c => c.member !== userId);
const spent = mapKey ? claims.some(c => c.byOwner || keyProofFor(mapKey, c.member) === c.keyProof) : claims.length > 0;
if (spent) throw new Error("Invite already used");
}
if (chain) {
for (const entry of chain) {
if (Array.isArray(entry.keys) && entry.keys.length) {
tribeCrypto.setKeys(entry.rootId, entry.keys, entry.gen || entry.keys.length);
} else if (entry.key) {
tribeCrypto.setKey(entry.rootId, entry.key, entry.gen || 1);
}
}
} else if (mapKey) {
ownCrypto.setKey(matched.rootId || matched.key, mapKey, matchedInvite.gen || 1);
}
if (!mapKey && matched.contentEncrypted && !matched.tribeId) throw new Error("Invalid or expired invite code");
await new Promise((resolve, reject) => ssbClient.publish({ type: "mapMember", target: rootId, member: userId, on: true, createdAt: new Date().toISOString(), ...(isPublic ? {} : { code: token }), ...(mapKey ? { keyProof: keyProofFor(mapKey, userId) } : {}) }, (e, r) => e ? reject(e) : resolve(r)));
if (tribeCrypto && mapKey) {
try {
const ssbKeys = require("../server/node_modules/ssb-keys");

68
src/models/market_model.js
View file

@ -3,7 +3,8 @@ const moment = require("../server/node_modules/moment")
const { getConfig } = require("../configs/config-manager.js")
const categories = require("../backend/opinion_categories")
const { buildValidatedTombstoneSet } = require('./tombstone_validator')
const { dedupeByPreferring, norm } = require('../backend/dedupe')
const { readTyped, memoIndex } = require('./typed_log')
const { dedupeByPreferring, norm, versionGroups } = require('../backend/dedupe')
const logLimit = getConfig().ssbLogStream?.limit || 1000
const N = (s) => String(s || "").toUpperCase().replace(/\s+/g, "_")
@ -63,11 +64,9 @@ module.exports = ({ cooler, tribeCrypto }) => {
return ssb
}
const readAll = async (ssbClient) => {
return new Promise((resolve, reject) =>
pull(ssbClient.createLogStream({ limit: logLimit }), pull.collect((err, msgs) => (err ? reject(err) : resolve(msgs))))
)
}
const MARKET_TYPES = ["market", "marketBid", "marketPurchase", "marketOpinion", "tombstone"]
const readAll = async (ssbClient) => readTyped(ssbClient, MARKET_TYPES, { limit: logLimit })
const buildMarketIndex = (messages) => {
const tomb = buildValidatedTombstoneSet(messages)
@ -83,31 +82,16 @@ module.exports = ({ cooler, tribeCrypto }) => {
if (c.type === "marketPurchase") { purchases.push({ target: c.target, author: m.value.author, ts: (m.value && m.value.timestamp) || 0 }); continue }
if (c.type === "marketOpinion" && c.target) { opinionMsgs.push({ target: c.target, author: m.value.author, category: c.category }); continue }
if (c.type !== "market") continue
nodes.set(m.key, { key: m.key, ts: (m.value && m.value.timestamp) || m.timestamp || 0, c, author: m.value.author })
nodes.set(m.key, { key: m.key, ts: (m.value && m.value.timestamp) || m.timestamp || 0, seq: m.value.sequence || 0, c, author: m.value.author })
}
const naivePrev = new Map(), strictNext = new Map()
for (const [key, n] of nodes) {
const t = n.c.replaces
if (!t) continue
naivePrev.set(key, t)
}
for (const [child, parent] of Array.from(naivePrev.entries())) {
const cn = nodes.get(child)
const pn = nodes.get(parent)
if (!pn) { naivePrev.delete(child); continue }
if (!cn || cn.author !== pn.author) { naivePrev.delete(child); nodes.delete(child); continue }
strictNext.set(parent, child)
}
return { tomb, nodes, bids, purchases, opinionMsgs, naivePrev, strictNext }
const vg = versionGroups(Array.from(nodes.values()).map(n => ({ key: n.key, author: n.author, seq: n.seq, ts: n.ts, replaces: typeof n.c.replaces === "string" ? n.c.replaces : null, claimedAuthor: typeof n.c.seller === "string" ? n.c.seller : null })))
return { tomb, nodes, bids, purchases, opinionMsgs, vg }
}
const resolveGroups = (idx) => {
const { tomb, nodes, bids, purchases, opinionMsgs, naivePrev, strictNext } = idx
const rootOf = (key) => { let x = key, g = 0; while (naivePrev.has(x) && nodes.has(naivePrev.get(x)) && g++ < 100000) x = naivePrev.get(x); return x }
const followStrict = (key) => { let x = key, g = 0; while (strictNext.has(x) && g++ < 100000) x = strictNext.get(x); return x }
const { tomb, nodes, bids, purchases, opinionMsgs, vg } = idx
const rootOf = vg.rootOf
const groups = new Map()
for (const key of nodes.keys()) { const r = rootOf(key); if (!groups.has(r)) groups.set(r, []); groups.get(r).push(key) }
const bidsByRoot = new Map()
for (const bd of bids) { if (!nodes.has(bd.target)) continue; const r = rootOf(bd.target); if (!bidsByRoot.has(r)) bidsByRoot.set(r, []); bidsByRoot.get(r).push(bd) }
const soldByRoot = new Map()
@ -128,13 +112,12 @@ module.exports = ({ cooler, tribeCrypto }) => {
}
const out = new Map()
for (const [root, keys] of groups) {
for (const root of vg.roots) {
const keys = vg.groups.get(root) || [root]
const rootNode = nodes.get(root)
const sellerId = rootNode ? rootNode.author : null
const sellerKeys = keys.filter(k => { const n = nodes.get(k); return n && n.author === sellerId })
let tip = followStrict(root)
const tipNode0 = nodes.get(tip)
if (!tipNode0 || tipNode0.author !== sellerId) tip = root
const tip = vg.tipOf(root)
if (tomb.has(tip)) continue
let best = nodes.get(tip)
if (!best) continue
@ -225,10 +208,9 @@ module.exports = ({ cooler, tribeCrypto }) => {
async resolveCurrentId(itemId) {
const ssbClient = await openSsb()
const messages = await readAll(ssbClient)
const { tomb, strictNext } = buildMarketIndex(messages)
let cur = itemId, g = 0
while (strictNext.has(cur) && g++ < 100000) cur = strictNext.get(cur)
if (tomb.has(cur)) throw new Error("Item not found")
const { tomb, vg } = buildMarketIndex(messages)
const cur = vg.tipOf(itemId)
if (tomb.has(cur) || vg.isForeignCopy(vg.rootOf(cur))) throw new Error("Item not found")
return cur
},
@ -431,9 +413,8 @@ module.exports = ({ cooler, tribeCrypto }) => {
const userId = ssbClient.id
const messages = await readAll(ssbClient)
const idx = buildMarketIndex(messages)
const rootOf = (key) => { let x = key, g = 0; while (idx.naivePrev.has(x) && idx.nodes.has(idx.naivePrev.get(x)) && g++ < 100000) x = idx.naivePrev.get(x); return x }
const grp = resolveGroups(idx).get(rootOf(itemId))
const idx = memoIndex('market', messages, buildMarketIndex)
const grp = resolveGroups(idx).get(idx.vg.rootOf(itemId))
if (!grp) return null
const { tip, rootId, best, statusN, poll, soldCount, buyers, opinions, voters } = grp
@ -495,6 +476,7 @@ module.exports = ({ cooler, tribeCrypto }) => {
const myId = ssbClient.id
const now = moment()
const list = Array.isArray(items) ? items : []
let changed = 0
for (const item of list) {
if (!item || !item.deadline) continue
@ -517,8 +499,10 @@ module.exports = ({ cooler, tribeCrypto }) => {
try {
await this.updateItemById(item.id, { status })
changed++
} catch (_) {}
}
return changed
},
async getItemByShopProductId(shopProductId) {
@ -532,11 +516,8 @@ module.exports = ({ cooler, tribeCrypto }) => {
const ssbClient = await openSsb()
const userId = ssbClient.id
const messages = await readAll(ssbClient)
const groups = resolveGroups(buildMarketIndex(messages))
let group = null
for (const g of groups.values()) {
if (g.rootId === itemId || g.tip === itemId) { group = g; break }
}
const idx = buildMarketIndex(messages)
const group = resolveGroups(idx).get(idx.vg.rootOf(itemId))
if (!group) throw new Error("Item not found")
if (group.best.author === userId) throw new Error("You cannot rate your own item")
if ((group.voters || []).includes(userId)) throw new Error("Already voted")
@ -595,8 +576,7 @@ module.exports = ({ cooler, tribeCrypto }) => {
const ssbClient = await openSsb()
const messages = await readAll(ssbClient)
const idx = buildMarketIndex(messages)
const rootOf = (key) => { let x = key, g = 0; while (idx.naivePrev.has(x) && idx.nodes.has(idx.naivePrev.get(x)) && g++ < 100000) x = idx.naivePrev.get(x); return x }
const grp = resolveGroups(idx).get(rootOf(itemId))
const grp = resolveGroups(idx).get(idx.vg.rootOf(itemId))
if (!grp) throw new Error("Item not found")
const c = grp.best.c

34
src/models/melody_model.js
View file

@ -22,12 +22,14 @@ const TYPE_TO_DEGREE = {
bookmark: 4,
feed: 5,
pad: 6,
room: 6,
chat: 7,
audio: 8,
video: 9,
image: 10,
document: 11,
torrent: 0,
file: 0,
map: 2,
pixelia: 3,
gameScore: 5,
@ -39,7 +41,35 @@ const TYPE_TO_DEGREE = {
courts: 0,
market: 1,
aiExchange: 6,
tombstone: 4
tombstone: 4,
wikiPage: 2,
emergency: 11,
emergencyConfirm: 11,
emergencyUpdate: 11,
mailingList: 5,
logisticsRoute: 3,
logisticsRating: 3,
podcast: 8,
podcastEpisode: 8,
podcastPlay: 8,
campaign: 9,
campaignSignature: 9,
campaignUpdate: 9
};
const FAMILY_DEGREE = [
['tribeParliament', 11], ['parliament', 11], ['courts', 0], ['school', 1], ['housing', 4], ['industry', 6],
['poll', 2], ['blog', 0], ['larp', 9], ['chat', 7], ['pad', 6], ['room', 6], ['wiki', 2], ['emergency', 11],
['mailing', 5], ['logistics', 3], ['podcast', 8], ['campaign', 9], ['project', 6], ['market', 1], ['shop', 1],
['job', 3], ['event', 8], ['task', 10], ['report', 10], ['votes', 7], ['calendar', 8], ['map', 2],
['transfer', 11], ['bank', 11], ['ubi', 11], ['feed', 5], ['tribe', 9], ['forum', 7], ['bookmark', 4],
['audio', 8], ['video', 9], ['image', 10], ['document', 11], ['torrent', 0], ['file', 0]
];
const degreeOf = (type) => {
if (TYPE_TO_DEGREE[type] != null) return TYPE_TO_DEGREE[type];
const family = FAMILY_DEGREE.find(([prefix]) => type.startsWith(prefix));
return family ? family[1] : (Math.abs(hashStr(type)) % 12);
};
const NOTE_FREQS = (() => {
@ -59,7 +89,7 @@ function blockToNote(msg) {
const c = msg && msg.value && msg.value.content;
if (!c || typeof c !== 'object') return null;
const type = String(c.type || '').trim() || 'unknown';
const degree = TYPE_TO_DEGREE[type] != null ? TYPE_TO_DEGREE[type] : (Math.abs(hashStr(type)) % 12);
const degree = degreeOf(type);
const size = Buffer.byteLength(JSON.stringify(msg.value), 'utf8');
const octIdx = size < 256 ? 0 : (size < 1024 ? 1 : 2);
const octave = OCTAVES[octIdx];

70
src/models/mentions_model.js
View file

@ -1,10 +1,11 @@
const fs = require('fs');
const pull = require('../server/node_modules/pull-stream');
const { readTyped, CONTENT_TYPES } = require('./typed_log');
const SEEN_FILE = 'oasis-mentions-seen';
const seenPath = () => {
try { return require('../server/ssb_config').statePath(SEEN_FILE); } catch (_) { return null; }
try { return require('../configs/state-manager').statePath(SEEN_FILE); } catch (_) { return null; }
};
const MAX_SEEN_KEYS = 500;
@ -32,6 +33,7 @@ const writeSeen = (keys, ts) => {
};
const { getConfig } = require('../configs/config-manager.js');
const { buildValidatedTombstoneSet } = require('./tombstone_validator');
const { versionGroups, versionEntry } = require('../backend/dedupe');
const logLimit = getConfig().ssbLogStream?.limit || 1000;
const TEXT_FIELDS = ['text', 'description', 'title', 'concept', 'question', 'rules', 'message', 'body', 'subject'];
@ -45,13 +47,7 @@ module.exports = ({ cooler }) => {
let ssb;
const openSsb = async () => { if (!ssb) ssb = await cooler.open(); return ssb; };
const getAllMessages = async (ssbClient) =>
new Promise((resolve, reject) => {
pull(
ssbClient.createLogStream({ limit: logLimit }),
pull.collect((err, msgs) => (err ? reject(err) : resolve(msgs)))
);
});
const getAllMessages = async (ssbClient) => readTyped(ssbClient, CONTENT_TYPES, { limit: logLimit, withWindow: true });
const mentionsList = (content, feedId) => {
const m = content && content.mentions;
@ -60,11 +56,24 @@ module.exports = ({ cooler }) => {
return values.some(v => v && (v.link === feedId || v === feedId));
};
const mentionsText = (content, feedId) => {
const escapeRe = (s) => String(s).replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
const nameRes = (names) => names.map(n => new RegExp(`(^|[^\\w\\u0080-\\uFFFF\\[\\]/@])@${escapeRe(n)}(?=$|[^\\w\\u0080-\\uFFFF/+=])`, 'i'));
const myNamesOf = (messages, feedId) => {
let latest = null;
for (const m of messages) {
const v = m && m.value;
const c = v && v.content;
if (!c || c.type !== 'about' || String(v.author) !== String(feedId) || c.about !== feedId || typeof c.name !== 'string' || !c.name.trim()) continue;
const ts = Number(v.timestamp || m.timestamp || 0);
if (!latest || ts >= latest.ts) latest = { ts, name: c.name.trim() };
}
return latest ? [latest.name] : [];
};
const mentionsText = (content, feedId, byName = []) => {
const bare = feedId.startsWith('@') ? feedId.slice(1) : feedId;
for (const field of TEXT_FIELDS) {
const v = content[field];
if (typeof v === 'string' && (v.includes(feedId) || v.includes(bare))) return true;
if (typeof v === 'string' && (v.includes(feedId) || v.includes(bare) || byName.some(re => re.test(v)))) return true;
}
if (Array.isArray(content.tags)) {
if (content.tags.some(t => typeof t === 'string' && (t.includes(feedId) || t.includes(bare)))) return true;
@ -80,6 +89,8 @@ module.exports = ({ cooler }) => {
return '';
};
const scanCache = { fingerprint: '', fresh: null };
const titleOf = (content) => {
for (const field of ['title', 'concept', 'question', 'subject']) {
const v = content[field];
@ -93,11 +104,17 @@ module.exports = ({ cooler }) => {
const ssbClient = await openSsb();
const myFeedId = ssbClient.id;
const messages = await getAllMessages(ssbClient);
const last = messages.length ? messages[messages.length - 1] : null;
const fingerprint = `${myFeedId}|${messages.length}|${last && last.key}`;
const fresh = (scanCache.fingerprint === fingerprint && scanCache.fresh)
? scanCache.fresh
: (() => {
const tomb = buildValidatedTombstoneSet(messages);
const vg = versionGroups(messages.filter(m => { const c = m && m.value && m.value.content; return c && typeof c === 'object' && c.type && !SKIP_TYPES.has(c.type); }).map(versionEntry));
const out = [];
const seen = new Set();
const replaced = new Set();
const byName = nameRes(myNamesOf(messages, myFeedId));
for (const m of messages) {
const v = m && m.value;
const c = v && v.content;
@ -108,8 +125,8 @@ module.exports = ({ cooler }) => {
if (Array.isArray(c.recps) || Array.isArray(v.content && v.content.recps)) continue;
if (c.tribeId && c.type !== 'tribe') continue;
if (String(v.author) === String(myFeedId)) continue;
if (typeof c.replaces === 'string') replaced.add(c.replaces);
if (!mentionsList(c, myFeedId) && !mentionsText(c, myFeedId)) continue;
if (vg.tipOf(m.key) !== m.key || !vg.roots.has(vg.rootOf(m.key))) continue;
if (!mentionsList(c, myFeedId) && !mentionsText(c, myFeedId, byName)) continue;
if (seen.has(m.key)) continue;
seen.add(m.key);
out.push({
@ -125,7 +142,10 @@ module.exports = ({ cooler }) => {
});
}
const fresh = out.filter(x => !replaced.has(x.key));
scanCache.fingerprint = fingerprint;
scanCache.fresh = out;
return out;
})();
const q = String(opts.q || '').trim().toLowerCase();
let list = q
? fresh.filter(x => x.text.toLowerCase().includes(q) || x.title.toLowerCase().includes(q) || x.type.toLowerCase().includes(q))
@ -140,6 +160,28 @@ module.exports = ({ cooler }) => {
seenAt: () => readSeen().ts,
readKeys: () => new Set(readSeen().keys.map(String)),
markRead(key) {
if (!key) return false;
const seen = readSeen();
const keys = [String(key)].concat(seen.keys.filter(k => String(k) !== String(key)));
return writeSeen(keys, seen.ts || Date.now());
},
markReadMany(list) {
const seen = readSeen();
const add = (Array.isArray(list) ? list : []).filter(Boolean).map(String);
const rest = seen.keys.filter(k => !add.includes(String(k)));
return writeSeen(add.concat(rest), seen.ts || Date.now());
},
markUnread(key) {
if (!key) return false;
const seen = readSeen();
return writeSeen(seen.keys.filter(k => String(k) !== String(key)), seen.ts);
},
markSeen(list) {
const items = Array.isArray(list) ? list : [];
const keys = items.map(x => x && x.key).filter(Boolean);

151
src/models/nextcloud_model.js Normal file
View file

@ -0,0 +1,151 @@
const fs = require('fs');
const path = require('path');
const { statePath } = require('../configs/state-manager');
const ACCOUNT_PATH = statePath('nextcloud-account.json');
const CHUNK_BYTES = 10 * 1024 * 1024;
const REQUEST_TIMEOUT_MS = 15000;
const CHUNK_TIMEOUT_MS = 10 * 60 * 1000;
const DEFAULT_FOLDER = 'Oasis';
const readAccount = () => {
try {
const a = JSON.parse(fs.readFileSync(ACCOUNT_PATH, 'utf8'));
return a && a.server && a.user && a.password ? a : null;
} catch (_) { return null; }
};
const writeAccount = (a) => {
fs.mkdirSync(path.dirname(ACCOUNT_PATH), { recursive: true });
fs.writeFileSync(ACCOUNT_PATH, JSON.stringify(a), { mode: 0o600 });
};
const normalizeServer = (v) => {
let s = String(v || '').trim();
if (!s) return '';
if (!/^https?:\/\//i.test(s)) s = `https://${s}`;
try {
const u = new URL(s);
if (!/^https?:$/.test(u.protocol)) return '';
return `${u.origin}${u.pathname.replace(/\/+$/, '')}`;
} catch (_) { return ''; }
};
const cleanFolder = (v) => String(v || '').trim().replace(/^\/+|\/+$/g, '').split('/').map(s => s.trim()).filter(Boolean).join('/') || DEFAULT_FOLDER;
const encPath = (p) => p.split('/').map(encodeURIComponent).join('/');
const safeName = (v) => String(v || '').replace(/[\\/:*?"<>|\r\n]+/g, ' ').trim().slice(0, 120) || 'file';
const pad = (n) => String(n).padStart(15, '0');
const readRange = async (file, start, end) => {
const fh = await fs.promises.open(file, 'r');
try {
const buf = Buffer.alloc(end - start);
let got = 0;
while (got < buf.length) {
const { bytesRead } = await fh.read(buf, got, buf.length - got, start + got);
if (!bytesRead) break;
got += bytesRead;
}
return got === buf.length ? buf : buf.subarray(0, got);
} finally { await fh.close(); }
};
module.exports = ({ isPublic = false } = {}) => {
const auth = (a) => 'Basic ' + Buffer.from(`${a.user}:${a.password}`).toString('base64');
const call = async (a, method, url, { headers = {}, body, timeout = REQUEST_TIMEOUT_MS } = {}) => {
const ctl = new AbortController();
const t = setTimeout(() => ctl.abort(), timeout);
try {
return await fetch(url, { method, headers: { Authorization: auth(a), 'OCS-APIRequest': 'true', ...headers }, body, signal: ctl.signal, redirect: 'manual' });
} finally { clearTimeout(t); }
};
const davFiles = (a) => `${a.server}/remote.php/dav/files/${encodeURIComponent(a.user)}`;
const davUploads = (a) => `${a.server}/remote.php/dav/uploads/${encodeURIComponent(a.user)}`;
const targetUrl = (a, name) => `${davFiles(a)}/${encPath(a.folder)}/${encodeURIComponent(name)}`;
const whoami = async (a) => {
let res;
try { res = await call(a, 'GET', `${a.server}/ocs/v1.php/cloud/user?format=json`); } catch (_) { throw new Error('nextcloudErrConnect'); }
if (!res.ok) throw new Error('nextcloudErrConnect');
let data = null;
try { data = await res.json(); } catch (_) {}
const d = data && data.ocs && data.ocs.data;
if (!d || !d.id) throw new Error('nextcloudErrConnect');
return { id: String(d.id), displayName: String(d['display-name'] || d.displayname || d.id) };
};
const mkcol = async (a, url) => {
const res = await call(a, 'MKCOL', url);
if (!(res.ok || res.status === 405)) throw new Error('nextcloudErrUpload');
};
const ensureFolder = async (a) => {
let cur = davFiles(a);
for (const seg of a.folder.split('/')) { cur += `/${encodeURIComponent(seg)}`; await mkcol(a, cur); }
};
const exists = async (a, url) => {
const res = await call(a, 'PROPFIND', url, { headers: { Depth: '0' } });
return res.status === 207 || res.status === 200;
};
const freeName = async (a, name) => {
if (!(await exists(a, targetUrl(a, name)))) return name;
const dot = name.lastIndexOf('.');
const stamp = Date.now().toString(36);
return dot > 0 ? `${name.slice(0, dot)}-${stamp}${name.slice(dot)}` : `${name}-${stamp}`;
};
const putWhole = async (a, filePath, size, name) => {
const res = await call(a, 'PUT', targetUrl(a, name), { body: await readRange(filePath, 0, size), headers: { 'Content-Type': 'application/octet-stream' }, timeout: CHUNK_TIMEOUT_MS });
if (!res.ok) throw new Error('nextcloudErrUpload');
};
const putChunked = async (a, filePath, size, name) => {
const base = `${davUploads(a)}/oasis-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 8)}`;
await mkcol(a, base);
for (let start = 0; start < size; start += CHUNK_BYTES) {
const end = Math.min(size, start + CHUNK_BYTES);
const res = await call(a, 'PUT', `${base}/${pad(start)}-${pad(end - 1)}`, { body: await readRange(filePath, start, end), headers: { 'Content-Type': 'application/octet-stream' }, timeout: CHUNK_TIMEOUT_MS });
if (!res.ok) throw new Error('nextcloudErrUpload');
}
const res = await call(a, 'MOVE', `${base}/.file`, { headers: { Destination: targetUrl(a, name), 'OC-Total-Length': String(size) }, timeout: CHUNK_TIMEOUT_MS });
if (!res.ok) throw new Error('nextcloudErrUpload');
};
const share = async (a, name) => {
const body = new URLSearchParams({ path: `/${a.folder}/${name}`, shareType: '3' });
const res = await call(a, 'POST', `${a.server}/ocs/v2.php/apps/files_sharing/api/v1/shares?format=json`, { headers: { 'Content-Type': 'application/x-www-form-urlencoded' }, body: body.toString() });
if (!res.ok) throw new Error('nextcloudErrShare');
let data = null;
try { data = await res.json(); } catch (_) {}
const url = data && data.ocs && data.ocs.data && data.ocs.data.url;
if (!url || !/^https?:\/\//i.test(String(url))) throw new Error('nextcloudErrShare');
return String(url);
};
return {
getAccount() {
const a = readAccount();
return a ? { server: a.server, user: a.user, folder: a.folder, displayName: a.displayName || a.user } : null;
},
async connect({ server, user, password, folder } = {}) {
if (isPublic) throw new Error('nextcloudErrConnect');
const a = { server: normalizeServer(server), user: String(user || '').trim(), password: String(password || ''), folder: cleanFolder(folder) };
if (!a.server || !a.user || !a.password) throw new Error('nextcloudErrConnect');
const me = await whoami(a);
a.displayName = me.displayName;
await ensureFolder(a);
writeAccount(a);
return this.getAccount();
},
disconnect() {
try { fs.unlinkSync(ACCOUNT_PATH); } catch (_) {}
},
async upload({ filePath, size, name } = {}) {
if (isPublic) throw new Error('nextcloudErrUpload');
const a = readAccount();
if (!a) throw new Error('nextcloudErrConnect');
const bytes = Number(size) || 0;
if (!filePath || !(bytes > 0)) throw new Error('nextcloudErrUpload');
const final = await freeName(a, safeName(name));
if (bytes <= CHUNK_BYTES) await putWhole(a, filePath, bytes, final);
else await putChunked(a, filePath, bytes, final);
const url = await share(a, final);
return { url, name: final, size: bytes };
}
};
};
module.exports.CHUNK_BYTES = CHUNK_BYTES;

76
src/models/onboarding_model.js
View file

@ -2,7 +2,8 @@ const fs = require('fs');
const path = require('path');
const pull = require('../server/node_modules/pull-stream');
const STEPS = ['language', 'ux', 'profile', 'federation', 'larp', 'greeting', 'backup'];
const STEPS = ['profile', 'ux', 'workflow', 'theme', 'community', 'greeting'];
const MARKED_STEPS = ['profile', 'ux', 'workflow', 'theme', 'community'];
const FLAG = 'oasis-first-contact';
const PENDING = 'welcome=pending';
const DISMISSED = 'welcome=dismissed';
@ -13,7 +14,13 @@ const ssbPathOf = (given) => {
try { return require('../server/ssb_config').path; } catch (_) { return null; }
};
const flagPath = (dir) => path.join(dir, FLAG);
const flagPath = (dir) => {
try {
const state = require('../configs/state-manager');
if (!dir || path.resolve(dir) === state.ssbDir()) return state.statePath(FLAG);
} catch (_) {}
return path.join(dir, FLAG);
};
function readFlag(dir) {
if (!dir) return { exists: false, id: '', markers: new Set() };
@ -49,6 +56,33 @@ const isPending = (dir) => {
const bannerVisible = (ssbPath) => isPending(ssbPathOf(ssbPath));
const DEFAULT_LANGUAGE = 'en';
const browserLanguage = (header, supported) => {
const list = Array.isArray(supported) ? supported : [];
const wanted = String(header || '').slice(0, 1024).split(',').map((part, i) => {
const [tag, ...params] = part.trim().split(';');
const qParam = params.map(p => p.trim()).find(p => /^q=/i.test(p));
const q = qParam ? Number(qParam.slice(2)) : 1;
return { tag: String(tag || '').trim().toLowerCase().replace(/_/g, '-'), q: Number.isFinite(q) ? q : 0, i };
}).filter(w => w.tag && w.tag !== '*' && w.q > 0).sort((a, b) => b.q - a.q || a.i - b.i);
for (const w of wanted) {
if (list.includes(w.tag)) return w.tag;
const base = w.tag.split('-')[0];
if (list.includes(base)) return base;
}
return '';
};
const firstRunLanguage = ({ fresh, isPublic, configured, cookie, header, supported } = {}) => {
const list = Array.isArray(supported) ? supported : [];
if (!fresh || isPublic) return { settled: true, language: '' };
if (list.includes(configured) && configured !== DEFAULT_LANGUAGE) return { settled: true, language: '' };
const picked = (list.includes(cookie) ? cookie : '') || browserLanguage(header, list);
if (!picked && !String(header || '').trim()) return { settled: false, language: '' };
return { settled: true, language: picked };
};
module.exports = ({ cooler, ssbPath } = {}) => {
const dir = () => ssbPathOf(ssbPath);
@ -56,7 +90,7 @@ module.exports = ({ cooler, ssbPath } = {}) => {
const ssb = await cooler.open();
const mine = ssb.id;
const all = await new Promise((resolve, reject) =>
pull(ssb.createLogStream({ reverse: true, limit: 2000 }), pull.collect((err, msgs) => err ? reject(err) : resolve(msgs || [])))
pull(ssb.createUserStream({ id: mine, reverse: true, limit: 2000 }), pull.collect((err, msgs) => err ? reject(err) : resolve(msgs || [])))
);
return { mine, messages: all.filter(m => m && m.value && m.value.author === mine) };
};
@ -72,16 +106,6 @@ module.exports = ({ cooler, ssbPath } = {}) => {
return c && typeof c === 'object' && c.type === 'feed' && String(c.text || '').trim().length > 0;
});
const hasJoinedLarp = (messages) => {
for (const m of messages) {
const c = m.value.content;
if (!c || typeof c !== 'object') continue;
if (c.type === 'larpJoinHouse' && c.house) return true;
if (c.type === 'larpLeaveLarp') return false;
}
return false;
};
const hasFederated = (mine, messages) => {
try {
const gossip = JSON.parse(fs.readFileSync(path.join(dir(), 'gossip.json'), 'utf8'));
@ -120,9 +144,20 @@ module.exports = ({ cooler, ssbPath } = {}) => {
}
},
adopt(feedId) {
const target = dir();
if (!target || !feedId) return false;
try {
fs.mkdirSync(target, { recursive: true });
fs.writeFileSync(flagPath(target), `${String(feedId)}\n${new Date().toISOString()}\n${DONE}\n`);
return true;
} catch (_) {
return false;
}
},
markStep(step) {
if (!STEPS.includes(step)) return false;
if (step !== 'language' && step !== 'backup' && step !== 'ux') return true;
if (!MARKED_STEPS.includes(step)) return false;
if (!isPending(dir())) return true;
return appendMarker(dir(), `step=${step}`);
},
@ -143,12 +178,11 @@ module.exports = ({ cooler, ssbPath } = {}) => {
let messages = [];
try { ({ mine, messages } = await ownMessages()); } catch (_) { mine = ''; messages = []; }
const steps = {
language: flag.markers.has('step=language'),
profile: hasProfile(mine, messages),
federation: hasFederated(mine, messages),
larp: hasJoinedLarp(messages),
profile: hasProfile(mine, messages) || flag.markers.has('step=profile'),
ux: flag.markers.has('step=ux'),
backup: flag.markers.has('step=backup'),
workflow: flag.markers.has('step=workflow'),
theme: flag.markers.has('step=theme'),
community: hasFederated(mine, messages) || flag.markers.has('step=community'),
greeting: hasGreeted(messages)
};
const profile = { id: mine, name: '', description: '', image: '' };
@ -180,3 +214,5 @@ module.exports = ({ cooler, ssbPath } = {}) => {
module.exports.bannerVisible = bannerVisible;
module.exports.STEPS = STEPS;
module.exports.browserLanguage = browserLanguage;
module.exports.firstRunLanguage = firstRunLanguage;

99
src/models/opinions_model.js
View file

@ -2,7 +2,9 @@ const pull = require('../server/node_modules/pull-stream');
const { getConfig } = require('../configs/config-manager.js');
const categories = require('../backend/opinion_categories');
const { buildValidatedTombstoneSet } = require('./tombstone_validator');
const { readTyped, CONTENT_TYPES, requestScope } = require('./typed_log');
const { buildVoteTally } = require('../backend/vote_tally');
const { versionGroups, versionEntry } = require('../backend/dedupe');
const logLimit = getConfig().ssbLogStream?.limit || 1000;
module.exports = ({ cooler }) => {
@ -22,8 +24,9 @@ module.exports = ({ cooler }) => {
const validTypes = [
'bookmark', 'votes', 'transfer',
'feed', 'image', 'audio', 'video', 'document', 'torrent',
'industry', 'project', 'report', 'task', 'event', 'shopProduct', 'housing', 'market'
'feed', 'image', 'audio', 'video', 'document', 'torrent', 'file',
'industry', 'project', 'report', 'task', 'event', 'shopProduct', 'housing', 'market', 'schoolCourse',
'podcast', 'podcastEpisode', 'campaign', 'logisticsRoute'
];
const getPreview = c => {
@ -31,25 +34,7 @@ module.exports = ({ cooler }) => {
return c.text || c.description || c.title || c.name || '';
};
const OPINION_MSG_TYPE = {
bookmark: 'bookmarkOpinion',
votes: 'votesOpinion',
transfer: 'transferOpinion',
feed: 'feedOpinion',
image: 'imageOpinion',
audio: 'audioOpinion',
video: 'videoOpinion',
document: 'documentOpinion',
torrent: 'torrentOpinion',
industry: 'industryOpinion',
project: 'projectOpinion',
report: 'reportOpinion',
task: 'taskOpinion',
event: 'eventOpinion',
shopProduct: 'shopOpinion',
housing: 'housingOpinion',
market: 'marketOpinion'
};
const OPINION_MSG_TYPE = categories.messageTypes;
const createVote = async (contentId, category) => {
const ssbClient = await openSsb();
@ -69,36 +54,23 @@ module.exports = ({ cooler }) => {
);
};
const listOpinions = async (filter = 'ALL', category = '') => {
const computeBase = async () => {
const ssbClient = await openSsb();
const userId = ssbClient.id;
const messages = await new Promise((res, rej) => {
pull(
ssbClient.createLogStream({ limit: logLimit }),
pull.collect((err, msgs) => err ? rej(err) : res(msgs))
);
});
const messages = await readTyped(ssbClient, CONTENT_TYPES, { limit: logLimit, withWindow: true });
const tombstoned = buildValidatedTombstoneSet(messages);
const replaces = new Map();
const byId = new Map();
const authorOf = new Map();
const opinionMsgs = [];
for (const msg of messages) {
const key = msg.key;
const c = msg.value?.content;
if (!c) continue;
if (c.type === 'tombstone') {
if (tombstoned.has(c.target)) byId.delete(c.target);
continue;
}
if (key) authorOf.set(key, msg.value.author);
if (c.type === 'tombstone') continue;
if (typeof c.type === 'string' && c.type.endsWith('Opinion') && c.target) {
opinionMsgs.push({ target: c.target, author: msg.value.author, category: c.category });
continue;
}
if (c.opinions && !tombstoned.has(key) && !['task', 'event', 'report'].includes(c.type)) {
if (c.replaces) replaces.set(c.replaces, key);
if (c.opinions && !['task', 'event', 'report'].includes(c.type)) {
byId.set(key, {
key,
value: {
@ -108,38 +80,19 @@ module.exports = ({ cooler }) => {
}
});
}
if (c.type === 'feed' && !tombstoned.has(key) && !byId.has(key)) {
if (c.replaces) replaces.set(c.replaces, key);
if (c.type === 'feed' && !byId.has(key)) {
byId.set(key, { key, value: { ...msg.value, content: c, preview: getPreview(c) } });
}
}
for (const [oldId, newId] of Array.from(replaces.entries())) {
const oldAuthor = authorOf.get(oldId);
const newM = byId.get(newId);
if (oldAuthor === undefined) { replaces.delete(oldId); continue; }
if (!newM || String(newM.value.author) !== String(oldAuthor)) {
replaces.delete(oldId);
byId.delete(newId);
}
const vg = versionGroups(Array.from(byId.values()).map(versionEntry));
for (const key of Array.from(byId.keys())) {
if (!vg.roots.has(vg.rootOf(key)) || vg.tipOf(key) !== key || tombstoned.has(key)) byId.delete(key);
}
for (const replacedId of replaces.keys()) {
byId.delete(replacedId);
}
const tipToChain = new Map();
for (const [oldId, newId] of replaces.entries()) {
let tip = newId; let g = 0;
while (replaces.has(tip) && g++ < 100000) tip = replaces.get(tip);
if (!tipToChain.has(tip)) tipToChain.set(tip, new Set([tip]));
tipToChain.get(tip).add(oldId); tipToChain.get(tip).add(newId);
}
const idToTip = new Map();
for (const [tip, chain] of tipToChain.entries()) for (const id of chain) idToTip.set(id, tip);
const opinionByTip = new Map();
for (const op of opinionMsgs) {
const tip = idToTip.get(op.target) || op.target;
const tip = vg.tipOf(op.target);
if (!opinionByTip.has(tip)) opinionByTip.set(tip, { opinions: {}, voters: new Set() });
const agg = opinionByTip.get(tip);
if (agg.voters.has(op.author)) continue;
@ -158,6 +111,12 @@ module.exports = ({ cooler }) => {
}
let filtered = Array.from(byId.values()).filter(m => validTypes.includes(m.value?.content?.type));
filtered = filtered.filter(m => {
const c = m.value?.content;
if (c?.type !== 'schoolCourse') return true;
if (c.encryptedPayload) return false;
return String(c.visibility || 'PUBLIC').toUpperCase() !== 'INVITE';
});
const blobTypes = ['document', 'image', 'audio', 'video'];
const blobCheckCache = new Map();
@ -237,10 +196,24 @@ module.exports = ({ cooler }) => {
}
filtered = Array.from(bySig.values());
filtered = filtered.filter(m => {
return filtered.filter(m => {
const ops = m.value?.content?.opinions || {};
return Object.values(ops).reduce((acc, x) => acc + (Number(x) || 0), 0) > 0;
});
};
const baseItems = () => {
const store = requestScope.getStore();
if (store && store.opinionsBase) return store.opinionsBase;
const pending = computeBase();
if (store) store.opinionsBase = pending;
return pending;
};
const listOpinions = async (filter = 'ALL', category = '') => {
const ssbClient = await openSsb();
const userId = ssbClient.id;
let filtered = (await baseItems()).slice();
if (filter === 'MINE') {
filtered = filtered.filter(m => m.value.author === userId);

210
src/models/pads_model.js
View file

@ -3,6 +3,7 @@ const crypto = require("crypto")
const fs = require("fs")
const { buildValidatedTombstoneSet } = require('./tombstone_validator')
const { collabContent, openInviteOf } = require('../backend/collab_content')
const { readTyped, memoIndex } = require('./typed_log')
const padCollab = collabContent({ membersField: 'members', undecField: 'undecryptable', contentFields: ['title', 'deadline', 'status'], listFields: ['tags', 'invites'] })
const path = require("path")
const { getConfig } = require("../configs/config-manager.js")
@ -59,6 +60,18 @@ module.exports = ({ cooler, cipherModel, tribeCrypto, padCrypto, tribesModel })
if (ownCrypto) return lookupKey(rootId)
try { return JSON.parse(fs.readFileSync(getLegacyKeyringPath(), "utf8"))[rootId] || null } catch (_) { return null }
}
const hashCode = (code, salt) => (ownCrypto && typeof ownCrypto.hashInviteCode === "function") ? ownCrypto.hashInviteCode(code, salt) : (tribeCrypto ? tribeCrypto.hashInviteCode(code, salt) : null)
const inviteMatches = (inv, token) => {
if (!token) return false
if (typeof inv === "string") return inv === token
if (!inv || typeof inv !== "object") return false
if (typeof inv.code === "string" && inv.code === token) return true
if (typeof inv.ch === "string" && (inv.ch === token || hashCode(token, inv.salt) === inv.ch)) return true
return false
}
const inviteToken = (inv) => (inv && typeof inv === "object" && typeof inv.ch === "string") ? inv.ch : (typeof inv === "string" ? inv : (inv && inv.code) || "")
const keyProofFor = (keyHex, id) => { try { return crypto.createHmac("sha256", Buffer.from(keyHex, "hex")).update(String(id), "utf8").digest("hex") } catch (_) { return null } }
const padKeys = (rootId) => (ownCrypto && typeof ownCrypto.getKeys === "function" ? ownCrypto.getKeys(rootId) : []).concat(tribeCrypto && typeof tribeCrypto.getKeys === "function" ? tribeCrypto.getKeys(rootId) : [])
const setPadKey = (rootId, keyHex) => {
migrateLegacyKeyring()
if (ownCrypto) { ownCrypto.setKey(rootId, keyHex, 1); return }
@ -117,29 +130,37 @@ module.exports = ({ cooler, cipherModel, tribeCrypto, padCrypto, tribesModel })
let tagsRaw = ""
try { deadline = c.deadline ? tryDecryptField(c.deadline, k) : "" } catch (_) {}
try { tagsRaw = c.tags ? tryDecryptField(c.tags, k) : "" } catch (_) {}
const ensureMemberKeys = async (ssbClient, messages, items) => {
if (!tribeCrypto) return
const distributed = new Map()
return { title: safeText(title), deadline, tags: normalizeTags(tagsRaw) }
} catch (_) {}
}
return null
}
const ensureMemberKeys = async (ssbClient, messages, items) => {
if (!tribeCrypto || !ownCrypto) return
const leavesByRoot = new Map()
for (const m of messages) {
const c = m.value && m.value.content
if (!c || c.type !== "tribe-keys" || !c.tribeId) continue
const mk = c.memberKeys
if (!mk || typeof mk !== "object") continue
if (!distributed.has(c.tribeId)) distributed.set(c.tribeId, new Set())
for (const id of Object.keys(mk)) distributed.get(c.tribeId).add(id)
if (!c || c.type !== "padMember" || c.on !== false || !c.target || !c.member) continue
if (!leavesByRoot.has(c.target)) leavesByRoot.set(c.target, [])
leavesByRoot.get(c.target).push({ member: c.member, author: m.value.author, ts: Number(m.timestamp || m.value.timestamp || 0) })
}
const ssbKeys = require("../server/node_modules/ssb-keys")
for (const item of (Array.isArray(items) ? items : [])) {
if (!item || item.undecryptable) continue
if (!item || item.undecryptable || item.author !== ssbClient.id) continue
const rootId = item.rootId
if (!rootId) continue
if (!rootId || !lookupKey(rootId)) continue
const members = Array.isArray(item.members) ? item.members : []
const leaves = (leavesByRoot.get(rootId) || []).filter(l => l.author === l.member || l.author === item.author)
const plan = ownCrypto.keyPlan({ rootId, ownerId: item.author, gen: lookupGen(rootId), members, messages, leaves })
if (plan.rotate) {
await rotatePadKey(rootId, members)
continue
}
if (!plan.missing.length) continue
const key = lookupKey(rootId)
if (!key) continue
const have = distributed.get(rootId) || new Set()
const missing = (Array.isArray(item.members) ? item.members : []).filter(m => m && m !== ssbClient.id && !have.has(m))
if (!missing.length) continue
const memberKeys = {}
for (const m of missing) {
for (const m of plan.missing) {
try { memberKeys[m] = tribeCrypto.boxKeyForMember(key, m, ssbKeys) } catch (_) {}
}
if (!Object.keys(memberKeys).length) continue
@ -149,12 +170,6 @@ module.exports = ({ cooler, cipherModel, tribeCrypto, padCrypto, tribesModel })
}
}
return { title: safeText(title), deadline, tags: normalizeTags(tagsRaw) }
} catch (_) {}
}
return null
}
const encryptForInvite = (padKeyHex, code, saltHex) => {
const salt = saltHex ? Buffer.from(saltHex, "hex") : Buffer.from(INVITE_SALT)
const derived = crypto.scryptSync(code, salt, 32)
@ -189,11 +204,10 @@ module.exports = ({ cooler, cipherModel, tribeCrypto, padCrypto, tribesModel })
if (!existing) return
const newKey = crypto.randomBytes(32).toString("hex")
const newGen = ownCrypto.addNewKey(rootId, newKey)
if (!Array.isArray(remainingMembers) || !remainingMembers.length) return
const ssbClient = await openSsb()
const ssbKeys = require("../server/node_modules/ssb-keys")
const memberKeys = {}
for (const m of remainingMembers) {
for (const m of new Set([ssbClient.id, ...(Array.isArray(remainingMembers) ? remainingMembers : [])])) {
try { memberKeys[m] = tribeCrypto.boxKeyForMember(newKey, m, ssbKeys) } catch (_) {}
}
if (Object.keys(memberKeys).length) {
@ -210,9 +224,12 @@ module.exports = ({ cooler, cipherModel, tribeCrypto, padCrypto, tribesModel })
const ssbKeys = require("../server/node_modules/ssb-keys")
const config = require("../server/ssb_config")
const msgs = await readAll(ssbClient)
const idx = buildIndex(msgs)
const trusted = (root, author) => { const n = idx.nodes.get(root); return !!author && !!n && (author === n.author || author === ssbClient.id) }
for (const m of msgs) {
const c = m.value && m.value.content
if (!c || c.type !== "tribe-keys") continue
if (!trusted(c.tribeId, m.value.author)) continue
const memberKeys = c.memberKeys
if (!memberKeys || typeof memberKeys !== "object") continue
const boxed = memberKeys[ssbClient.id]
@ -226,18 +243,15 @@ module.exports = ({ cooler, cipherModel, tribeCrypto, padCrypto, tribesModel })
} catch (_) {}
}
const readAll = async (ssbClient) =>
new Promise((resolve, reject) =>
pull(ssbClient.createLogStream({ limit: logLimit }), pull.collect((err, msgs) => err ? reject(err) : resolve(msgs)))
)
const PAD_TYPES = ["pad", "padMember", "padEntry", "tribe-keys", "tombstone"]
const readAll = async (ssbClient) => readTyped(ssbClient, PAD_TYPES, { limit: logLimit })
const buildIndex = (messages) => {
const tomb = new Set()
const nodes = new Map()
const parent = new Map()
const child = new Map()
const strictParent = new Map()
const strictChild = new Map()
const authorByKey = new Map()
const tombRequests = []
const memberMsgs = []
@ -248,19 +262,14 @@ module.exports = ({ cooler, cipherModel, tribeCrypto, padCrypto, tribesModel })
const c = v.content
if (!c) continue
if (c.type === "tombstone" && c.target) { tombRequests.push({ target: c.target, author: v.author }); continue }
if (c.type === "padMember" && c.target) { memberMsgs.push({ target: c.target, member: c.member, on: c.on !== false, author: v.author, ts: v.timestamp || m.timestamp || 0, code: typeof c.code === "string" ? c.code : "" }); continue }
if (c.type === "padMember" && c.target) { memberMsgs.push({ target: c.target, member: c.member, on: c.on !== false, author: v.author, ts: v.timestamp || m.timestamp || 0, code: typeof c.code === "string" ? c.code : "", keyProof: typeof c.keyProof === "string" ? c.keyProof : "" }); continue }
if (c.type === "pad") {
nodes.set(k, { key: k, ts: v.timestamp || m.timestamp || 0, c, author: v.author })
nodes.set(k, { key: k, ts: v.timestamp || m.timestamp || 0, seq: v.sequence || 0, c, author: v.author })
authorByKey.set(k, v.author)
if (c.replaces) { parent.set(k, c.replaces); child.set(c.replaces, k) }
}
}
for (const [k, node] of nodes) {
const t = node.c.replaces
if (t) { const orig = nodes.get(t); if (orig && orig.author === node.author) { strictParent.set(k, t); strictChild.set(t, k) } }
}
for (const t of tombRequests) {
const targetAuthor = authorByKey.get(t.target)
if (targetAuthor && t.author === targetAuthor) tomb.add(t.target)
@ -268,53 +277,86 @@ module.exports = ({ cooler, cipherModel, tribeCrypto, padCrypto, tribesModel })
const rootOf = (id) => { let cur = id, g = 0; while (parent.has(cur) && g++ < 100000) cur = parent.get(cur); return cur }
const tipOf = (id) => { let cur = id, g = 0; while (child.has(cur) && g++ < 100000) cur = child.get(cur); return cur }
const strictRootOf = (id) => { let cur = id, g = 0; while (strictParent.has(cur) && g++ < 100000) cur = strictParent.get(cur); return cur }
const contentTipOf = (root) => {
const rn = nodes.get(root)
if (!rn) return root
let cur = root, best = root, g = 0
const seen = new Set()
while (child.has(cur) && !seen.has(cur) && g++ < 100000) {
seen.add(cur)
const next = child.get(cur)
const n = nodes.get(next)
if (!n) break
if (n.author === rn.author && !tomb.has(next)) best = next
cur = next
}
return best
const rootMemo = new Map()
const groupRootOf = (id) => {
if (rootMemo.has(id)) return rootMemo.get(id)
let cur = id, g = 0
while (parent.has(cur) && nodes.has(parent.get(cur)) && g++ < 100000) cur = parent.get(cur)
rootMemo.set(id, cur)
return cur
}
const ownerTip = new Map()
for (const [k, n] of nodes) {
const r = groupRootOf(k)
const rn = nodes.get(r)
if (!rn || n.author !== rn.author) continue
const prev = ownerTip.has(r) ? nodes.get(ownerTip.get(r)) : null
if (!prev || n.seq > prev.seq || (n.seq === prev.seq && n.ts >= prev.ts)) ownerTip.set(r, k)
}
const contentTipOf = (root) => ownerTip.get(root) || root
const roots = new Set()
for (const id of nodes.keys()) roots.add(strictRootOf(id))
for (const id of nodes.keys()) roots.add(groupRootOf(id))
const invitesOf = (root) => {
const out = []
for (const id of [root, contentTipOf(root)]) { const n = nodes.get(id); if (n && Array.isArray(n.c.invites)) out.push(...n.c.invites) }
return out
}
const selfJoinAllowed = (root, mm) => {
const oc = (nodes.get(root) || {}).c || {}
if (oc.encrypted !== true || oc.tribeId) return true
const invites = invitesOf(root)
if (invites.some(inv => inv && typeof inv === "object" && inv.public === true)) return true
if (mm.keyProof && padKeys(root).some(k => keyProofFor(k, mm.member) === mm.keyProof)) return true
return !!mm.code && invites.some(inv => typeof inv === "string" ? inv === mm.code : !!(inv && typeof inv === "object" && typeof inv.code === "string" && inv.code === mm.code))
}
const memberByRoot = new Map()
const consumedByRoot = new Map()
const codeClaimsByRoot = new Map()
for (const mm of memberMsgs) {
if (!nodes.has(mm.target)) continue
const r = strictRootOf(mm.target)
const r = groupRootOf(mm.target)
const ownerAuthor = nodes.get(r) && nodes.get(r).author
if (!ownerAuthor) continue
const self = mm.member === mm.author
const byOwner = mm.author === ownerAuthor
if (!self && !byOwner) continue
if (!mm.member) continue
if (!memberByRoot.has(r)) memberByRoot.set(r, new Map())
const m2 = memberByRoot.get(r)
const p = m2.get(mm.member)
if (!p || mm.ts >= p.ts) m2.set(mm.member, { on: mm.on, ts: mm.ts })
if (mm.on && mm.code) {
if (!codeClaimsByRoot.has(r)) codeClaimsByRoot.set(r, [])
codeClaimsByRoot.get(r).push({ code: mm.code, member: mm.member, keyProof: mm.keyProof, byOwner })
}
if (self && !byOwner && mm.on && !selfJoinAllowed(r, mm)) continue
if (mm.on && mm.code) {
if (!consumedByRoot.has(r)) consumedByRoot.set(r, new Set())
consumedByRoot.get(r).add(mm.code)
}
if (!memberByRoot.has(r)) memberByRoot.set(r, new Map())
const m2 = memberByRoot.get(r)
const p = m2.get(mm.member)
if (!p || mm.ts >= p.ts) m2.set(mm.member, { on: mm.on, ts: mm.ts })
}
const codeClaims = (root, codes) => (codeClaimsByRoot.get(root) || []).filter(c => codes.includes(c.code))
const isCodeConsumed = (root, code) => {
if (!code) return false
const set = consumedByRoot.get(root)
return !!(set && set.has(code))
}
for (const [k, n] of nodes) {
const r = groupRootOf(k)
const rn = nodes.get(r)
if (!rn || n.author === rn.author || !Array.isArray(n.c.members) || !selfJoinAllowed(r, { member: n.author })) continue
if (!memberByRoot.has(r)) memberByRoot.set(r, new Map())
const m2 = memberByRoot.get(r)
const p = m2.get(n.author)
if (!p || n.ts >= p.ts) m2.set(n.author, { on: n.c.members.includes(n.author), ts: n.ts })
}
const resolveMembers = (root) => {
const ownerNode = nodes.get(root)
const oc = ownerNode ? ownerNode.c : {}
@ -326,7 +368,7 @@ module.exports = ({ cooler, cipherModel, tribeCrypto, padCrypto, tribesModel })
const tipByRoot = new Map()
for (const r of roots) tipByRoot.set(r, contentTipOf(r))
return { tomb, nodes, parent, child, rootOf, tipOf, strictRootOf, contentTipOf, tipByRoot, resolveMembers, isCodeConsumed }
return { tomb, nodes, parent, child, rootOf, tipOf, groupRootOf, contentTipOf, tipByRoot, resolveMembers, isCodeConsumed, codeClaims }
}
const decryptPadFields = (c, rootId, tribeKeys) => {
@ -358,7 +400,7 @@ module.exports = ({ cooler, cipherModel, tribeCrypto, padCrypto, tribesModel })
status: c.status || "OPEN",
deadline,
tags,
author: c.author || node.author,
author: node.author,
members: Array.isArray(members) ? members : (Array.isArray(c.members) ? c.members : []),
invites: Array.isArray(c.invites) ? c.invites : [],
createdAt: c.createdAt || new Date(node.ts).toISOString(),
@ -423,7 +465,7 @@ module.exports = ({ cooler, cipherModel, tribeCrypto, padCrypto, tribesModel })
const ssbClient = await openSsb()
const messages = await readAll(ssbClient)
const idx = buildIndex(messages)
const root = idx.strictRootOf(id)
const root = idx.groupRootOf(id)
const tip = idx.contentTipOf(root)
if (idx.tomb.has(tip)) throw new Error("Not found")
return root
@ -433,7 +475,7 @@ module.exports = ({ cooler, cipherModel, tribeCrypto, padCrypto, tribesModel })
const ssbClient = await openSsb()
const messages = await readAll(ssbClient)
const idx = buildIndex(messages)
const root = idx.strictRootOf(id)
const root = idx.groupRootOf(id)
const tip = idx.contentTipOf(root)
if (idx.tomb.has(tip)) throw new Error("Not found")
return tip
@ -518,7 +560,7 @@ module.exports = ({ cooler, cipherModel, tribeCrypto, padCrypto, tribesModel })
return new Promise(async (resolve, reject) => {
ssbClient.get(tipId, async (err, item) => {
if (err || !item?.content) return reject(new Error("Pad not found"))
if (item.content.author !== userId) return reject(new Error("Not the author"))
if (item.author !== userId) return reject(new Error("Not the author"))
const c = item.content
const isEncrypted = c.encrypted === true
let keyHex = null
@ -564,7 +606,7 @@ module.exports = ({ cooler, cipherModel, tribeCrypto, padCrypto, tribesModel })
return new Promise(async (resolve, reject) => {
ssbClient.get(tipId, async (err, item) => {
if (err || !item?.content) return reject(new Error("Pad not found"))
if (item.content.author !== userId) return reject(new Error("Not the author"))
if (item.author !== userId) return reject(new Error("Not the author"))
const c = item.content
let keyHex = null
let usesTribeKey = false
@ -599,13 +641,11 @@ module.exports = ({ cooler, cipherModel, tribeCrypto, padCrypto, tribesModel })
if (!pad) throw new Error("Pad not found")
if (pad.author === userId) throw new Error("Author cannot leave their own pad")
if (!Array.isArray(pad.members) || !pad.members.includes(userId)) return
const members = pad.members.filter(m => m !== userId)
const rootId = await this.resolveRootId(padId)
await new Promise((resolve, reject) => {
const content = { type: "padMember", target: rootId, member: userId, on: false, createdAt: new Date().toISOString() }
ssbClient.publish(content, (e) => e ? reject(e) : resolve())
})
try { await rotatePadKey(rootId, members) } catch (_) {}
},
async ingestKeys() { await ingestOwnTribeKeys() },
@ -640,7 +680,8 @@ module.exports = ({ cooler, cipherModel, tribeCrypto, padCrypto, tribesModel })
const key = tryDecryptPublicInviteKey(c.invites)
if (key) setPadKey(rootId, key)
}
const content = { type: "padMember", target: rootId, member: feedId, on: true, createdAt: new Date().toISOString(), ...(typeof consumedCode === "string" && consumedCode ? { code: consumedCode } : {}) }
const proofKey = feedId === ssbClient.id && c.encrypted === true && !c.tribeId ? getPadKey(rootId) : null
const content = { type: "padMember", target: rootId, member: feedId, on: true, createdAt: new Date().toISOString(), ...(typeof consumedCode === "string" && consumedCode ? { code: consumedCode } : {}), ...(proofKey ? { keyProof: keyProofFor(proofKey, feedId) } : {}) }
ssbClient.publish(content, (e, res) => e ? reject(e) : resolve(res))
})
})
@ -653,7 +694,7 @@ module.exports = ({ cooler, cipherModel, tribeCrypto, padCrypto, tribesModel })
return new Promise((resolve, reject) => {
ssbClient.get(tipId, (err, item) => {
if (err || !item?.content) return reject(new Error("Pad not found"))
if (item.content.author !== userId) return reject(new Error("Not the author"))
if (item.author !== userId) return reject(new Error("Not the author"))
const tombstone = { type: "tombstone", target: tipId, deletedAt: new Date().toISOString(), author: userId }
ssbClient.publish(tombstone, (e) => e ? reject(e) : resolve())
})
@ -663,8 +704,8 @@ module.exports = ({ cooler, cipherModel, tribeCrypto, padCrypto, tribesModel })
async getPadById(id) {
const ssbClient = await openSsb()
const messages = await readAll(ssbClient)
const idx = buildIndex(messages)
const root = idx.strictRootOf(id)
const idx = memoIndex('pads', messages, buildIndex)
const root = idx.groupRootOf(id)
const tip = idx.contentTipOf(root)
if (idx.tomb.has(tip)) return null
const node = idx.nodes.get(tip)
@ -703,12 +744,13 @@ module.exports = ({ cooler, cipherModel, tribeCrypto, padCrypto, tribesModel })
const code = crypto.randomBytes(INVITE_BYTES).toString("hex")
let invite = code
const pubFlag = opts.public ? { public: true } : {}
const inviteSalt = generateInviteSalt()
if (keyHex) {
const inviteSalt = generateInviteSalt()
const ek = encryptForInvite(keyHex, code, inviteSalt)
invite = { code, ek, salt: inviteSalt, ...pubFlag }
invite = opts.public ? { code, ek, salt: inviteSalt, ...pubFlag } : { ch: hashCode(code, inviteSalt), ek, salt: inviteSalt }
} else {
invite = opts.public ? { code, public: true } : { ch: hashCode(code, inviteSalt), salt: inviteSalt }
}
if (opts.public && typeof invite !== "object") invite = { code, public: true }
const invites = [...pad.invites, invite]
await this.updatePadById(padId, { invites })
return code
@ -734,18 +776,18 @@ module.exports = ({ cooler, cipherModel, tribeCrypto, padCrypto, tribesModel })
if (pad.author !== userId) throw new Error("Only the author can remove invites")
const invites = (Array.isArray(pad.invites) ? pad.invites : []).filter(inv => !(typeof inv === "object" && inv.public === true))
await this.updatePadById(padId, { invites })
if (!pad.tribeId) await rotatePadKey(await this.resolveRootId(padId), pad.members)
},
async joinByInvite(code) {
const ssbClient = await openSsb()
const userId = ssbClient.id
const pads = await this.listAll()
const pads = await collectPads(buildIndex(await readAll(ssbClient)))
let matchedPad = null
let matchedInvite = null
for (const p of pads) {
for (const inv of p.invites) {
if (typeof inv === "string" && inv === code) { matchedPad = p; matchedInvite = inv; break }
if (typeof inv === "object" && inv.code === code) { matchedPad = p; matchedInvite = inv; break }
if (inviteMatches(inv, code)) { matchedPad = p; matchedInvite = inv; break }
}
if (matchedPad) break
}
@ -753,17 +795,19 @@ module.exports = ({ cooler, cipherModel, tribeCrypto, padCrypto, tribesModel })
if (matchedPad.members.includes(userId)) throw new Error("Already a member")
const isPublic = typeof matchedInvite === "object" && matchedInvite.public === true
let resolvedRootId = await this.resolveRootId(matchedPad.rootId)
const token = inviteToken(matchedInvite) || code
let padKey = null
if (typeof matchedInvite === "object" && matchedInvite.ek) padKey = decryptFromInvite(matchedInvite.ek, code, matchedInvite.salt)
if (!isPublic) {
const messages = await readAll(ssbClient)
const idx = buildIndex(messages)
if (idx.isCodeConsumed(resolvedRootId, code)) throw new Error("Invite already used")
if (idx.isCodeConsumed(resolvedRootId, token) || idx.isCodeConsumed(resolvedRootId, code)) throw new Error("Invite already used")
const claims = idx.codeClaims(resolvedRootId, [token, code]).filter(c => c.member !== userId)
const spent = padKey ? claims.some(c => c.byOwner || keyProofFor(padKey, c.member) === c.keyProof) : claims.length > 0
if (spent) throw new Error("Invite already used")
}
let padKey = null
if (typeof matchedInvite === "object" && matchedInvite.ek) {
padKey = decryptFromInvite(matchedInvite.ek, code, matchedInvite.salt)
setPadKey(resolvedRootId, padKey)
}
await this.addMemberToPad(matchedPad.rootId, userId, isPublic ? "" : code)
if (padKey) setPadKey(resolvedRootId, padKey)
await this.addMemberToPad(matchedPad.rootId, userId, isPublic ? "" : token)
if (tribeCrypto && padKey && resolvedRootId) {
try {
const ssbKeys = require("../server/node_modules/ssb-keys")
@ -854,7 +898,7 @@ module.exports = ({ cooler, cipherModel, tribeCrypto, padCrypto, tribesModel })
}
entries.push({
key: m.key,
author: c.author || v.author,
author: v.author,
text,
createdAt: c.createdAt || new Date(v.timestamp || 0).toISOString()
})

2
src/models/panicmode_model.js
View file

@ -1,11 +1,9 @@
const os = require('os');
const fs = require('fs');
const path = require('path');
module.exports = {
removeSSB: async () => {
try {
// ruta de la cuenta activa: con varias identidades, ~/.ssb fijo borraria otra
const ssbPath = require('../server/ssb_config').path;
await fs.promises.rm(ssbPath, { recursive: true, force: true });
} catch (error) {

1166
src/models/parliament_model.js
View file

File diff suppressed because it is too large Load diff

103
src/models/peer_health.js Normal file
View file

@ -0,0 +1,103 @@
const fs = require('fs');
const FAILURES_MIN = 3;
const DEAD_AFTER_MS = 7 * 24 * 60 * 60 * 1000;
const RETRY_CAP_MS = 3 * 60 * 60 * 1000;
const FEED_ID = /^@[A-Za-z0-9+/]{43}=\.ed25519$/;
const healthPath = () => require('../configs/state-manager').statePath('peer-health.json');
let cache = null;
let cachePath = null;
const canonicalKey = (key) => {
let k = String(key || '').trim().replace(/-/g, '+').replace(/_/g, '/');
if (!k) return null;
if (!k.startsWith('@')) k = '@' + k;
if (!k.endsWith('.ed25519')) k += '.ed25519';
if (!/=\.ed25519$/.test(k)) k = k.replace(/\.ed25519$/, '=.ed25519');
return FEED_ID.test(k) ? k : null;
};
const load = () => {
const p = healthPath();
if (!cache || cachePath !== p) {
cachePath = p;
try { cache = JSON.parse(fs.readFileSync(p, 'utf8')) || {}; } catch (_) { cache = {}; }
if (typeof cache !== 'object' || Array.isArray(cache)) cache = {};
}
return cache;
};
const save = () => { try { fs.writeFileSync(cachePath, JSON.stringify(cache)); } catch (_) {} };
const observe = ({ connected = [], failing = [], now = Date.now() } = {}) => {
const h = load();
let changed = false;
const up = new Set(connected.map(canonicalKey).filter(Boolean));
for (const id of up) {
if (h[id]) { delete h[id]; changed = true; }
}
for (const k of failing) {
const id = canonicalKey(k);
if (!id || up.has(id) || h[id]) continue;
h[id] = { failingSince: now };
changed = true;
}
if (changed) save();
return changed;
};
const isDead = ({ key, failures = 0, now = Date.now() } = {}) => {
if (!(Number(failures) >= FAILURES_MIN)) return false;
const e = load()[canonicalKey(key)] || {};
const observed = Number(e.failingSince) ? now - Number(e.failingSince) : 0;
return Math.max(observed, Number(failures) * RETRY_CAP_MS) >= DEAD_AFTER_MS;
};
const addressOf = (item) => {
const m = Array.isArray(item) ? String(item[0] || '').match(/^(?:net|onion):([^:~]+):(\d+)/) : null;
if (m) return `${m[1]}:${m[2]}`;
const data = Array.isArray(item) ? item[1] : item;
return data && data.host ? `${data.host}:${Number(data.port) || 8008}` : null;
};
const failuresByKey = (...sources) => {
const byKey = new Map();
for (const list of sources) {
for (const item of list || []) {
const data = Array.isArray(item) ? item[1] : item;
const id = data && canonicalKey(data.key);
if (!id) continue;
const f = Number(data.failure) || 0;
const e = byKey.get(id) || { any: 0, addrs: new Map() };
const where = addressOf(item);
if (where) e.addrs.set(where, Math.max(e.addrs.get(where) || 0, f));
else e.any = Math.max(e.any, f);
byKey.set(id, e);
}
}
const out = new Map();
for (const [id, e] of byKey) out.set(id, e.addrs.size ? Math.min(...[...e.addrs.values()].map(f => Math.max(f, e.any))) : e.any);
return out;
};
const classifyNetError = (err, address = '') => {
const msg = `${(err && err.code) || ''} ${(err && err.message) || err || ''}`.toLowerCase();
const onion = /\.onion/i.test(String(address || ''));
if (msg.includes('network paused')) return 'paused';
if (onion && /only know|could not connect to/.test(msg)) return 'tor';
if (/127\.0\.0\.1:9[01]50/.test(msg) || (onion && /econnrefused|socks/.test(msg) && !/ttl|hostunreachable|host unreachable|general/.test(msg))) return 'tor';
if (/shs\.client: (error when expecting server to accept challenge|server responded with invalid challenge)|application cap/.test(msg)) return 'keys';
if (/shs\.client: (server hung up when we sent hello|the server's response accepting)/.test(msg)) return 'identity';
if (/econnrefused/.test(msg)) return 'refused';
if (/enotfound|eai_again|getaddrinfo/.test(msg)) return 'notfound';
if (/enetunreach|ehostunreach|ehostdown|enetdown|hostunreachable|host unreachable|networkunreachable/.test(msg)) return 'unreachable';
if (/timed? ?out|etimedout|ttl/.test(msg)) return 'timeout';
if (/econnreset|epipe|hung up|aborted|closed/.test(msg)) return 'dropped';
return 'other';
};
const NET_REASON_KEYS = {
paused: 'peerErrPaused', tor: 'peerErrTor', keys: 'peerErrKeys', identity: 'peerErrIdentity', refused: 'peerErrRefused',
notfound: 'peerErrNotFound', unreachable: 'peerErrUnreachable', timeout: 'peerErrTimeout', dropped: 'peerErrDropped', other: 'peerErrOther', silent: 'peerErrSilent'
};
module.exports = { canonicalKey, observe, isDead, failuresByKey, classifyNetError, NET_REASON_KEYS, DEAD_AFTER_MS };

325
src/models/phone_model.js Normal file
View file

@ -0,0 +1,325 @@
const fs = require('fs');
const pull = require('../server/node_modules/pull-stream');
const sharedState = require('../configs/shared-state');
const { notify: desktopNotify, i18nNow, displayName } = require('../backend/desktopNotify');
const { statePath } = require('../configs/state-manager');
const HISTORY_MAX = 200;
const HEARD_MAX = 1000;
const LATEST_MAX = 5;
const PUBS_TTL_MS = 5 * 60 * 1000;
const RECOUNT_DELAY_MS = 500;
const FEED_ID = /^@[A-Za-z0-9+/]{43}=\.ed25519$/;
const readJson = (p, fallback) => { try { return JSON.parse(fs.readFileSync(p, 'utf8')); } catch (_) { return fallback; } };
const writeJson = (p, data) => { try { fs.writeFileSync(p, JSON.stringify(data, null, 2)); } catch (_) {} };
const asP = (fn, ...args) => new Promise((resolve, reject) => fn(...args, (err, v) => err ? reject(err) : resolve(v)));
const rpcValue = (fn, ...args) => new Promise((resolve) => {
try {
const r = fn(...args, (err, v) => resolve(err ? null : v));
if (r && typeof r.then === 'function') r.then(resolve, () => resolve(null));
else if (r !== undefined) resolve(r);
} catch (_) { resolve(null); }
});
module.exports = ({ cooler, pmModel, nameOf, isPublic, encryptFile, isAvailable, prefetch }) => {
const historyFile = statePath('phone-history.json');
const seenFile = statePath('phone-seen.json');
let ssb = null;
let subscribed = false;
let pubsAt = 0;
let pubsTimer = null;
const open = async () => { if (!ssb) ssb = await cooler.open(); return ssb; };
const phone = async () => { const s = await open(); return s && s.phone ? s.phone : null; };
const history = () => { const h = readJson(historyFile, []); return Array.isArray(h) ? h : []; };
const seen = () => {
const s = readJson(seenFile, {});
return { missedSeenAt: Number(s.missedSeenAt) || 0, heard: Array.isArray(s.heard) ? s.heard : [] };
};
const refreshCount = async () => {
let n = 0;
const st = sharedState.getPhoneCall ? sharedState.getPhoneCall() : null;
if (st && st.dir === 'in' && st.phase === 'incoming') n += 1;
const sn = seen();
n += history().filter(h => h && h.outcome === 'missed' && h.endedAt > sn.missedSeenAt).length;
try {
const me = (await open()).id;
const heard = new Set(sn.heard);
n += (await pmModel.listPams()).filter(m => m.value.author !== me && !heard.has(m.key)).length;
} catch (_) {}
sharedState.setPhoneCount(n);
return n;
};
const notify = async (peer, callId) => {
const i18n = i18nNow();
let name = '';
try { name = await nameOf(peer); } catch (_) {}
desktopNotify(i18n.phoneTitle, `${displayName(peer, name)} ${i18n.phoneIsCalling}`, callId ? `call:${callId}` : null, {
timeoutMs: require('../server/phone_module').RING_MS,
actions: [{ key: 'answer', label: i18n.phoneAnswer }, { key: 'reject', label: i18n.phoneReject }],
onAction: async (chosen) => {
try {
const ph = await phone();
const st = ph ? await rpcValue(ph.state) : null;
if (!st || st.id !== callId || st.phase !== 'incoming') return;
await act(chosen === 'answer' ? 'accept' : 'reject');
} catch (_) {}
}
});
};
const ROOM_EVENT_TEXT = { join: 'roomEventJoined', leave: 'roomEventLeft', recStart: 'roomEventRecStart', recStop: 'roomEventRecStop', mute: 'roomEventMuted', unmute: 'roomEventUnmuted', hand: 'roomEventHand' };
const roomNotices = { rid: null, since: 0, seen: new Set() };
const announceRoom = async (room) => {
if (!room || !room.rid) { roomNotices.rid = null; roomNotices.seen = new Set(); return; }
if (roomNotices.rid !== room.rid) { roomNotices.rid = room.rid; roomNotices.since = Number(room.joinedAt) || Date.now(); roomNotices.seen = new Set(); }
if (!room.notify) { roomNotices.since = Date.now() + 1; return; }
const events = Array.isArray(room.events) ? room.events : [];
const keyOf = (e) => `${e.ts}:${e.t}:${e.id}`;
const fresh = events.filter(e => e && ROOM_EVENT_TEXT[e.t] && !roomNotices.seen.has(keyOf(e)) && Number(e.ts) >= roomNotices.since);
roomNotices.seen = new Set(events.filter(Boolean).map(keyOf));
if (!fresh.length) return;
let me = null;
try { me = (await open()).id; } catch (_) {}
const i18n = i18nNow();
for (const e of fresh) {
if (e.id === me) continue;
let name = '';
try { name = await nameOf(e.id); } catch (_) {}
desktopNotify(i18n.roomsTitle, String(i18n[ROOM_EVENT_TEXT[e.t]] || '{name}').replace('{name}', displayName(e.id, name)));
}
};
const subscribe = async () => {
if (subscribed) return;
const ph = await phone();
if (!ph || subscribed) return;
subscribed = true;
pull(ph.events(), pull.drain((ev) => {
if (!ev) return;
if (ev.type === 'state') {
sharedState.setPhoneCall(ev.state || null);
refreshCount();
} else if (ev.type === 'room') {
sharedState.setPhoneRoom(ev.room || null);
announceRoom(ev.room).catch(() => {});
} else if (ev.type === 'recordLimit') {
finishPam().catch(() => {});
} else if (ev.type === 'incoming') {
notify(ev.peer, ev.id);
} else if (ev.type === 'ended' && ev.call) {
const h = history();
h.unshift(ev.call);
writeJson(historyFile, h.slice(0, HISTORY_MAX));
refreshCount();
}
}, () => {
subscribed = false;
const retry = setTimeout(() => { subscribe().catch(() => {}); }, 5000);
if (retry.unref) retry.unref();
}));
try { sharedState.setPhoneCall((await rpcValue(ph.state)) || null); } catch (_) {}
try { if (typeof ph.roomState === 'function') sharedState.setPhoneRoom((await rpcValue(ph.roomState)) || null); } catch (_) {}
};
let arrivalsWatched = false;
let recountTimer = null;
const scheduleRecount = () => {
if (recountTimer) return;
recountTimer = setTimeout(() => { recountTimer = null; refreshCount().catch(() => {}); }, RECOUNT_DELAY_MS);
if (recountTimer.unref) recountTimer.unref();
};
const watchArrivals = async () => {
if (arrivalsWatched) return;
const s = await open();
if (!s || !s.db || typeof s.db.onMsgAdded !== 'function') return;
arrivalsWatched = true;
s.db.onMsgAdded((ev) => {
const v = ev && ev.kvt && ev.kvt.value;
if (!v || v.author === s.id || typeof v.content !== 'string') return;
scheduleRecount();
}, false);
};
const act = async (method, ...args) => {
await subscribe();
const ph = await phone();
if (!ph) throw new Error('unavailable');
return asP(ph[method], ...args);
};
const finishPam = async () => {
const ph = await phone();
if (!ph) return false;
const out = await asP(ph.recordStop);
if (!out || !out.file) return false;
try {
const enc = await encryptFile(out.file);
await pmModel.sendPam(out.peer, enc.cipher, Number(out.durationMs) / 1000);
return true;
} finally {
fs.unlink(out.file, () => {});
}
};
const findPam = async (key) => (await pmModel.listPams()).find(m => m.key === key) || null;
const byType = (s, type) => new Promise((resolve) => {
if (!s || typeof s.messagesByType !== 'function') return resolve([]);
try { pull(s.messagesByType({ type }), pull.collect((err, msgs) => resolve(err ? [] : msgs))); } catch (_) { resolve([]); }
});
const defaultPubId = () => {
const m = String((require('../configs/config-manager').defaultInvite() || {}).code || '').match(/(@[A-Za-z0-9+/]{43}=\.ed25519)/);
return m ? m[1] : '';
};
let announcedPubs = new Set();
const refreshPubs = async (force = false) => {
const ids = new Set();
const add = (id) => { if (FEED_ID.test(String(id || ''))) ids.add(String(id)); };
try {
const s = await open();
if (force || Date.now() - pubsAt >= PUBS_TTL_MS) {
pubsAt = Date.now();
const found = new Set();
for (const m of await byType(s, 'pub')) { const v = m && m.value; const k = v && v.content && v.content.address && v.content.address.key; if (FEED_ID.test(String(k || '')) && (v.author === k || v.author === s.id)) found.add(String(k)); }
for (const m of await byType(s, 'pubAvailability')) { const k = m && m.value && m.value.author; if (FEED_ID.test(String(k || ''))) found.add(String(k)); }
announcedPubs = found;
}
add(defaultPubId());
for (const id of announcedPubs) add(id);
try { const ph = s.phone; if (ph && typeof ph.pubs === 'function') for (const id of (await rpcValue(ph.pubs)) || []) add(id); } catch (_) {}
ids.delete(s.id);
} catch (_) { add(defaultPubId()); }
sharedState.setPubIds(ids);
return ids;
};
const isPub = (id) => sharedState.isPubId(id);
return {
async start() {
await subscribe();
await watchArrivals().catch(() => {});
await refreshCount();
await refreshPubs(true);
if (!pubsTimer) { pubsTimer = setInterval(() => { refreshPubs(true).catch(() => {}); }, PUBS_TTL_MS); if (pubsTimer.unref) pubsTimer.unref(); }
},
refreshCount,
refreshPubs,
isPub,
async available() {
if (isPublic) return false;
const ph = await phone();
return !!(ph && await rpcValue(ph.available));
},
async state() {
await subscribe();
const ph = await phone();
return ph ? rpcValue(ph.state) : null;
},
async call(to, opts = {}) {
await refreshPubs();
const list = Array.isArray(to) ? to : [to];
if (list.some(isPub)) throw new Error('pub');
return act('call', list.length === 1 ? list[0] : list, { legacy: Array.isArray(opts.legacy) ? opts.legacy : [] });
},
accept: () => act('accept'),
reject: () => act('reject'),
hangup: () => act('end'),
mute: (flag) => act('mute', !!flag),
camera: (flag) => act('camera', !!flag),
async videoFrames(peer) {
const ph = await phone();
return ph && typeof ph.videoFrames === 'function' ? ph.videoFrames({ peer: String(peer || '') }) : null;
},
silence: (id, flag) => act('silence', { id, on: !!flag }),
dismiss: () => act('dismiss'),
recordCancel: () => act('recordCancel'),
async sendPam() { await subscribe(); return finishPam(); },
history,
async markAllHeard(keys) {
const s = seen();
const set = new Set(s.heard);
for (const k of keys || []) if (typeof k === 'string' && k) set.add(k);
s.heard = [...set].slice(-HEARD_MAX);
writeJson(seenFile, s);
refreshCount();
},
deleteHistory(ids) {
const drop = new Set((ids || []).map(String));
if (!drop.size) return;
writeJson(historyFile, history().filter(h => !drop.has(String(h && h.id))));
},
latestCalls(items) {
const me = ssb && ssb.id;
const out = [];
const seenPeers = new Set();
for (const h of items || history()) {
for (const peer of (Array.isArray(h.peers) && h.peers.length ? h.peers : [h.peer])) {
if (!peer || peer === me || seenPeers.has(peer) || isPub(peer)) continue;
seenPeers.add(peer);
out.push({ id: peer, dir: h.dir, outcome: h.outcome, startedAt: h.startedAt });
if (out.length >= LATEST_MAX) return out;
}
}
return out;
},
async pams() {
const me = (await open()).id;
const heard = new Set(seen().heard);
const validShare = (s) => !!s && typeof s === 'object' && typeof s.key === 'string' && typeof s.manifestBlobId === 'string';
const list = (await pmModel.listPams())
.filter(m => m.value.author !== me && validShare(m.value.content && m.value.content.share))
.map(m => ({ key: m.key, from: m.value.author, sentAt: m.value.content.sentAt || new Date(m.timestamp || 0).toISOString(), durationSec: Number(m.value.content.durationSec) || 0, heard: heard.has(m.key), share: m.value.content.share, ready: true }))
.sort((a, b) => String(b.sentAt).localeCompare(String(a.sentAt)));
if (typeof isAvailable === 'function') {
for (const item of list) {
item.ready = await isAvailable(item.share).catch(() => false);
if (!item.ready && typeof prefetch === 'function') prefetch(item.share).catch(() => {});
}
}
return list.map(({ share, ...rest }) => rest);
},
async pamCipher(key) {
const m = await findPam(key);
const share = m && m.value && m.value.content && m.value.content.share;
if (!share || typeof share.key !== 'string' || typeof share.manifestBlobId !== 'string') return null;
return share;
},
markHeard(key) {
const s = seen();
if (s.heard.includes(key)) return false;
s.heard.push(key);
s.heard = s.heard.slice(-HEARD_MAX);
writeJson(seenFile, s);
refreshCount();
return true;
},
async deletePam(key) {
const m = await findPam(key);
if (!m) return null;
return pmModel.deleteMessageById(key);
},
async knownFeeds() {
const s = await open();
await refreshPubs();
const hops = await new Promise((resolve) => {
try { s.friends.hops({ start: s.id, max: 2 }, (err, h) => resolve(err ? {} : (h || {}))); } catch (_) { resolve({}); }
});
const out = new Map();
for (const [id, hop] of Object.entries(hops)) {
if (id === s.id || isPub(id) || !(Number(hop) >= 0)) continue;
out.set(id, Number(hop));
}
return out;
},
markMissedSeen() {
const s = seen();
s.missedSeenAt = Date.now();
writeJson(seenFile, s);
refreshCount();
}
};
};

28
src/models/phone_number.js Normal file
View file

@ -0,0 +1,28 @@
const crypto = require('crypto');
const FEED_ID = /^@[A-Za-z0-9+/]{43}=\.ed25519$/;
const MSG_ID = /^%[A-Za-z0-9+/]{43}=?\.sha256$/;
const digitsOf = (id) => {
const n = crypto.createHash('sha256').update(String(id)).digest().readUInt32BE(0) % 1000000;
const s = String(n).padStart(6, '0');
return `${s.slice(0, 3)}-${s.slice(3)}`;
};
const phoneNumberOf = (id) => (FEED_ID.test(String(id || '')) ? digitsOf(id) : null);
const roomNumberOf = (id) => (MSG_ID.test(String(id || '')) ? digitsOf(id) : null);
const normalizeNumber = (value) => {
const digits = String(value || '').replace(/[^0-9A-Za-z]/g, '');
return /^\d{6}$/.test(digits) ? `${digits.slice(0, 3)}-${digits.slice(3)}` : null;
};
const resolveNumber = (value, known) => {
const number = normalizeNumber(value);
if (!number) return [];
return [...(known || new Map())]
.filter(([id]) => phoneNumberOf(id) === number)
.map(([id, hop]) => ({ id, hop: Number(hop) }))
.sort((a, b) => a.hop - b.hop);
};
module.exports = { phoneNumberOf, roomNumberOf, normalizeNumber, resolveNumber };

121
src/models/pixelia_model.js
View file

@ -1,8 +1,49 @@
const pull = require('../server/node_modules/pull-stream');
const { getConfig } = require('../configs/config-manager.js');
const { buildValidatedTombstoneSet } = require('./tombstone_validator');
const { readTyped } = require('./typed_log');
const logLimit = getConfig().ssbLogStream?.limit || 1000;
const PIXELIA_TYPES = ['pixelia', 'tombstone'];
const COLOR_RE = /^#[0-9a-fA-F]{6}$/;
const pixelCoordinate = (c) => {
if (!Number.isInteger(c.x) || !Number.isInteger(c.y) || c.x < 0 || c.x > 49 || c.y < 0 || c.y > 199) return null;
const key = `${c.x + 1}:${c.y + 1}`;
if (c.coordinateKey != null && c.coordinateKey !== key) return null;
return key;
};
const resolvePixels = (messages) => {
const tombstoned = buildValidatedTombstoneSet(messages);
const byKey = new Map();
for (const m of messages) {
const c = m.value?.content;
if (!c || c.type !== 'pixelia' || !m.key) continue;
const coordinateKey = pixelCoordinate(c);
if (!coordinateKey || typeof c.color !== 'string' || !COLOR_RE.test(c.color)) continue;
byKey.set(m.key, { m, coordinateKey });
}
const replaced = new Set();
for (const { m, coordinateKey } of byKey.values()) {
const prev = byKey.get(m.value.content.replaces);
if (prev && prev.coordinateKey === coordinateKey) replaced.add(prev.m.key);
}
const latest = new Map();
for (const [k, { m, coordinateKey }] of byKey) {
if (replaced.has(k) || tombstoned.has(k)) continue;
const cur = latest.get(coordinateKey);
const ts = m.value.timestamp || 0;
const curTs = cur ? cur.value.timestamp || 0 : 0;
if (!cur || ts > curTs || (ts === curTs && k > cur.key)) latest.set(coordinateKey, m);
}
return latest;
};
module.exports = ({ cooler }) => {
let ssb;
@ -13,52 +54,22 @@ module.exports = ({ cooler }) => {
const getPixelByCoordinate = async (coordinateKey) => {
const ssbClient = await openSsb();
const messages = await new Promise((res, rej) => {
pull(
ssbClient.createLogStream({ limit: logLimit }),
pull.collect((err, msgs) => err ? rej(err) : res(msgs))
);
});
const tombstoned = buildValidatedTombstoneSet(messages);
const replaces = new Map();
const byId = new Map();
for (const m of messages) {
const c = m.value?.content;
const k = m.key;
if (!c || c.type !== 'pixelia' || c.coordinateKey !== coordinateKey) continue;
if (tombstoned.has(k)) continue;
if (c.replaces) replaces.set(c.replaces, k);
byId.set(k, m);
}
for (const [replacedId, replacingId] of replaces.entries()) {
const orig = byId.get(replacedId);
if (!orig) continue;
const rep = byId.get(replacingId);
if (!rep || rep.value.author !== orig.value.author) {
byId.delete(replacingId);
continue;
}
byId.delete(replacedId);
}
return [...byId.values()][0] || null;
const messages = await readTyped(ssbClient, PIXELIA_TYPES, { limit: logLimit });
return resolvePixels(messages).get(coordinateKey) || null;
};
const paintPixel = async (x, y, color) => {
if (x < 1 || x > 50 || y < 1 || y > 200) {
if (!Number.isInteger(x) || !Number.isInteger(y) || x < 1 || x > 50 || y < 1 || y > 200) {
throw new Error('Coordinates out of bounds. Please use x (1-50) and y (1-200)');
}
if (typeof color !== 'string' || !COLOR_RE.test(color)) throw new Error('Invalid color');
const ssbClient = await openSsb();
const userId = ssbClient.id;
const coordinateKey = `${x}:${y}`;
const existingPixel = await getPixelByCoordinate(coordinateKey);
if (existingPixel) {
if (existingPixel && existingPixel.value.author === userId) {
const tombstone = {
type: 'tombstone',
target: existingPixel.key,
@ -69,7 +80,7 @@ module.exports = ({ cooler }) => {
);
}
const contributors = existingPixel?.value?.content?.contributors_inhabitants || [];
const contributors = Array.isArray(existingPixel?.value?.content?.contributors_inhabitants) ? existingPixel.value.content.contributors_inhabitants : [];
const contributors_inhabitants = contributors.includes(userId)
? contributors
: [...contributors, userId];
@ -93,46 +104,14 @@ module.exports = ({ cooler }) => {
const listPixels = async () => {
const ssbClient = await openSsb();
const messages = await new Promise((res, rej) => {
pull(
ssbClient.createLogStream({ limit: logLimit }),
pull.collect((err, msgs) => err ? rej(err) : res(msgs))
);
});
const messages = await readTyped(ssbClient, PIXELIA_TYPES, { limit: logLimit });
const tombstoned = buildValidatedTombstoneSet(messages);
const replaces = new Map();
const byKey = new Map();
for (const m of messages) {
const c = m.value?.content;
const k = m.key;
if (!c) continue;
if (c.type === 'tombstone') continue;
if (c.type === 'pixelia') {
if (tombstoned.has(k)) continue;
if (c.replaces) replaces.set(c.replaces, k);
byKey.set(k, m);
}
}
for (const [replacedId, replacingId] of replaces.entries()) {
const orig = byKey.get(replacedId);
if (!orig) continue;
const rep = byKey.get(replacingId);
if (!rep || rep.value.author !== orig.value.author) {
byKey.delete(replacingId);
continue;
}
byKey.delete(replacedId);
}
return Array.from(byKey.values()).map(m => ({
return Array.from(resolvePixels(messages).values()).map(m => ({
x: m.value.content.x + 1,
y: m.value.content.y + 1,
color: m.value.content.color,
author: m.value.author,
contributors_inhabitants: m.value.content.contributors_inhabitants || [],
contributors_inhabitants: [...new Set([...(Array.isArray(m.value.content.contributors_inhabitants) ? m.value.content.contributors_inhabitants : []).filter(id => typeof id === 'string'), m.value.author])],
timestamp: m.value.timestamp
}));
};

196
src/models/pm_model.js
View file

@ -1,5 +1,89 @@
const pull = require('../server/node_modules/pull-stream');
const util = require('../server/node_modules/util');
const util = require('util');
const longText = require('../backend/long_text');
const fs = require('fs');
const INBOX_READ_FILE = 'oasis-inbox-read';
const INBOX_ARCHIVE_FILE = 'oasis-inbox-archived';
const flagPath = (name) => {
try { return require('../configs/state-manager').statePath(name); } catch (_) { return null; }
};
const readFlagSet = (name) => {
const file = flagPath(name);
if (!file) return new Set();
try { const arr = JSON.parse(fs.readFileSync(file, 'utf8')); return new Set(Array.isArray(arr) ? arr.map(String) : []); } catch (_) { return new Set(); }
};
const writeFlagSet = (name, set) => {
const file = flagPath(name);
if (!file) return false;
try { fs.writeFileSync(file, JSON.stringify(Array.from(set))); return true; } catch (_) { return false; }
};
const readKeys = () => readFlagSet(INBOX_READ_FILE);
const markRead = (key) => { if (!key) return false; const set = readKeys(); set.add(String(key)); return writeFlagSet(INBOX_READ_FILE, set); };
const markUnread = (key) => { if (!key) return false; const set = readKeys(); set.delete(String(key)); return writeFlagSet(INBOX_READ_FILE, set); };
const markReadMany = (keys) => { const set = readKeys(); for (const k of (keys || [])) { if (k) set.add(String(k)); } return writeFlagSet(INBOX_READ_FILE, set); };
const archivedKeys = () => readFlagSet(INBOX_ARCHIVE_FILE);
const archive = (key) => { if (!key) return false; const set = archivedKeys(); set.add(String(key)); return writeFlagSet(INBOX_ARCHIVE_FILE, set); };
const unarchive = (key) => { if (!key) return false; const set = archivedKeys(); set.delete(String(key)); return writeFlagSet(INBOX_ARCHIVE_FILE, set); };
const INBOX_BOTS = {
blogs: ['BLOG_NEW'],
jobs: ['JOB_MATCH', 'JOB_SUBSCRIBED', 'JOB_UNSUBSCRIBED'],
projects: ['PROJECT_FOLLOWED', 'PROJECT_UNFOLLOWED', 'PROJECT_PLEDGE'],
market: ['MARKET_SOLD'],
shops: ['SHOP_SOLD'],
political: ['LARP_RULING', 'PARLIAMENT_GOV', 'TRIBE_GOV'],
banking: ['BANKING_UBI_PAID', 'BANKING_UBI_AVAILABLE', 'BANKING_CONFIRM_PENDING', 'WALLET_PAYMENT'],
school: ['SCHOOL_ENROLLED', 'SCHOOL_INVITED', 'SCHOOL_ADMITTED', 'SCHOOL_CERTIFICATE', 'SCHOOL_PASSED', 'SCHOOL_LESSON_NEW'],
industry: ['INDUSTRY_ADMITTED', 'INDUSTRY_APPLICATION', 'INDUSTRY_INVITED', 'INDUSTRY_DISSOLVED', 'INDUSTRY_BUILD_APPROVED', 'INDUSTRY_DISTRIBUTED'],
housing: ['HOUSING_REQUESTED', 'HOUSING_CANCELLED', 'HOUSING_UNAVAILABLE'],
wiki: ['WIKI_EDITED', 'WIKI_RESTORED'],
emergencies: ['EMERGENCY_UPDATED', 'EMERGENCY_RESOLVED'],
podcasts: ['PODCAST_EPISODE'],
campaigns: ['CAMPAIGN_UPDATED', 'CAMPAIGN_ACHIEVED', 'CAMPAIGN_RAISED'],
logistics: ['LOGISTICS_UPDATED', 'LOGISTICS_CLOSED', 'LOGISTICS_BOOKED', 'LOGISTICS_CANCELLED', 'LOGISTICS_CONFIRMED', 'LOGISTICS_REJECTED', 'LOGISTICS_DELIVERED']
};
const BOT_BY_SUBJECT = new Map();
for (const [bot, subjects] of Object.entries(INBOX_BOTS)) for (const s of subjects) BOT_BY_SUBJECT.set(s, bot);
const isReminderSubject = (s) => /^(Task Reminder:|Calendar Reminder:)/i.test(String(s || ''));
const botOf = (content) => {
const c = content || {};
if (isReminderSubject(c.subject)) return 'reminders';
if (c.meta && c.meta.type === 'project-pledge') return 'projects';
return BOT_BY_SUBJECT.get(String(c.subject || '').toUpperCase()) || null;
};
const mutedBots = (cfg) => {
const c = cfg || {};
return new Set(Array.isArray(c.inboxMutedBots) ? c.inboxMutedBots.map(String) : []);
};
const privateCaches = new WeakMap();
const privateCacheFor = (ssb, userId) => {
let byUser = privateCaches.get(ssb);
if (!byUser) { byUser = new Map(); privateCaches.set(ssb, byUser); }
let c = byUser.get(userId);
if (!c) {
c = { lastTs: 0, seen: new Set(), posts: new Map(), pams: new Map(), chunks: new Map(), tombClaims: new Map(), authorByKey: new Map(), recpsByKey: new Map(), version: 0 };
byUser.set(userId, c);
}
return c;
};
const tombedKeys = (cache, userId) => {
const tombed = new Set();
for (const [target, tombAuthors] of cache.tombClaims.entries()) {
const origAuthor = cache.authorByKey.get(target);
const origRecps = cache.recpsByKey.get(target) || [];
for (const tombAuthor of tombAuthors) {
if (tombAuthor === origAuthor || tombAuthor === userId || origRecps.includes(tombAuthor)) {
tombed.add(target);
break;
}
}
}
return tombed;
};
module.exports = ({ cooler }) => {
let ssb;
@ -26,12 +110,35 @@ module.exports = ({ cooler }) => {
}
return {
readKeys,
markRead,
markUnread,
markReadMany,
archivedKeys,
archive,
unarchive,
botOf,
mutedBots,
INBOX_BOTS,
type: 'post',
async sendToMany(recipients = [], subject = '', text = '', crypter = false, ref = '') {
const others = uniqueRecps(recipients).filter(id => id !== userId);
if (!others.length) return [];
const out = [];
for (let i = 0; i < others.length; i += 6) {
const batch = others.slice(i, i + 6);
const r = await this.sendMessage(batch, subject, text, crypter, ref).catch(() => null);
if (r) out.push(r);
}
return out;
},
async sendMessage(recipients = [], subject = '', text = '', crypter = false, ref = '') {
const ssbClient = await openSsb();
const recps = uniqueRecps([userId, ...recipients]);
const content = {
const publishAsync = util.promisify(ssbClient.private.publish);
const content = await longText.chunkContent({
type: 'post',
from: userId,
to: recps,
@ -41,8 +148,7 @@ module.exports = ({ cooler }) => {
private: true,
...(ref ? { ref: String(ref) } : {}),
...(crypter ? { crypter: true } : {})
};
const publishAsync = util.promisify(ssbClient.private.publish);
}, 'text', { maxBytes: longText.CONTENT_BYTES.boxed, extra: { private: true }, publish: (c) => publishAsync(c, recps) });
return publishAsync(content, recps);
},
@ -119,20 +225,20 @@ module.exports = ({ cooler }) => {
async listAllPrivate(opts = {}) {
const includeDeleted = !!(opts && opts.includeDeleted);
const ssbClient = await openSsb();
const cache = privateCacheFor(ssbClient, userId);
const raw = await new Promise((resolve, reject) => {
pull(
ssbClient.createLogStream({ reverse: false }),
ssbClient.createLogStream(cache.lastTs > 0 ? { reverse: false, gte: cache.lastTs } : { reverse: false }),
pull.collect((err, arr) => err ? reject(err) : resolve(arr))
);
});
const posts = [];
const tombed = new Set();
const tombClaims = new Map();
const authorByKey = new Map();
const recpsByKey = new Map();
for (const m of raw) {
if (!m || !m.value) continue;
const keyIn = m.key || m.value?.key || m.value?.hash || '';
if (!keyIn || cache.seen.has(keyIn)) continue;
cache.seen.add(keyIn);
const recvTs = Number(m.timestamp) || 0;
if (recvTs > cache.lastTs) cache.lastTs = recvTs;
const valueIn = m.value || m;
const tsIn = m.timestamp || m.value?.timestamp || Date.now();
let dec;
@ -146,18 +252,23 @@ module.exports = ({ cooler }) => {
const k = dec?.key || keyIn;
if (!c || c.private !== true || !k) continue;
if (c.type === 'tombstone' && c.target) {
const set = tombClaims.get(c.target) || new Set();
const set = cache.tombClaims.get(c.target) || new Set();
set.add(v.author);
tombClaims.set(c.target, set);
cache.tombClaims.set(c.target, set);
continue;
}
authorByKey.set(k, v.author);
if (c.type === 'post') {
if (c.type === longText.CHUNK_TYPE) {
const ch = longText.chunkOf(v.author, c);
if (ch) cache.chunks.set(k, ch);
continue;
}
cache.authorByKey.set(k, v.author);
if (c.type === 'post' || c.type === 'pam') {
const to = Array.isArray(c.to) ? c.to : [];
recpsByKey.set(k, to);
cache.recpsByKey.set(k, to);
const author = v.author;
if (author === userId || to.includes(userId)) {
posts.push({
(c.type === 'pam' ? cache.pams : cache.posts).set(k, {
key: k,
value: { author, content: c },
timestamp: v.timestamp || tsIn
@ -165,17 +276,48 @@ module.exports = ({ cooler }) => {
}
}
}
for (const [target, tombAuthors] of tombClaims.entries()) {
const origAuthor = authorByKey.get(target);
const origRecps = recpsByKey.get(target) || [];
for (const tombAuthor of tombAuthors) {
if (tombAuthor === origAuthor || tombAuthor === userId || origRecps.includes(tombAuthor)) {
tombed.add(target);
break;
}
}
}
return posts.filter(m => m && m.key && (includeDeleted || !tombed.has(m.key)));
cache.version += raw.length ? 1 : 0;
const tombed = tombedKeys(cache, userId);
const lookup = longText.lookupIn(cache.chunks);
return Array.from(cache.posts.values())
.filter(m => m && m.key && (includeDeleted || !tombed.has(m.key)))
.map(m => longText.hasChunks(m.value.content)
? { ...m, value: { ...m.value, content: longText.resolveField(m.value.content, 'text', m.value.author, lookup) } }
: m);
},
async listPams() {
await this.listAllPrivate();
const cache = privateCacheFor(ssb, userId);
const tombed = tombedKeys(cache, userId);
return Array.from(cache.pams.values()).filter(m => m && m.key && !tombed.has(m.key));
},
async sendPam(recipient, share, durationSec) {
const ssbClient = await openSsb();
if (!share || typeof share !== 'object') throw new Error('Invalid audio');
const recps = uniqueRecps([userId, recipient]);
const content = {
type: 'pam',
from: userId,
to: recps,
share,
durationSec: Math.max(0, Math.round(Number(durationSec) || 0)),
sentAt: new Date().toISOString(),
private: true
};
const publishAsync = util.promisify(ssbClient.private.publish);
return publishAsync(content, recps);
},
privateCacheVersion() {
return ssb ? privateCacheFor(ssb, userId).version : 0;
}
};
};
module.exports.botOf = botOf;
module.exports.mutedBots = mutedBots;
module.exports.INBOX_BOTS = INBOX_BOTS;
module.exports.readKeys = readKeys;
module.exports.archivedKeys = archivedKeys;

329
src/models/podcasts_model.js Normal file
View file

@ -0,0 +1,329 @@
const { readTyped, memoIndex } = require('./typed_log');
const { buildValidatedTombstoneSet } = require('./tombstone_validator');
const { getConfig } = require('../configs/config-manager.js');
const categories = require('../backend/opinion_categories');
const logLimit = getConfig().ssbLogStream?.limit || 1000;
const CHANNEL_TYPE = 'podcast';
const EPISODE_TYPE = 'podcastEpisode';
const OPINION_TYPE = 'podcastOpinion';
const PLAY_TYPE = 'podcastPlay';
const TYPES = [CHANNEL_TYPE, EPISODE_TYPE, OPINION_TYPE, PLAY_TYPE, 'tombstone'];
const CATEGORIES = ['NEWS', 'MUSIC', 'TALK', 'EDUCATION', 'TECH', 'CULTURE', 'COMMUNITY', 'OASIS', 'OTHER'];
const RECENT_MS = 24 * 60 * 60 * 1000;
const MEDIA_RE = /!?\[(image|audio|video):([^\]]*)\]\((&[^)]+)\)/;
const safeText = (v) => String(v == null ? '' : v).trim();
const normU = (v) => String(v || '').trim().toUpperCase();
const normalizeCategory = (v) => (CATEGORIES.includes(normU(v)) ? normU(v) : 'OTHER');
const normalizeTags = (raw) => {
if (!raw) return [];
const arr = Array.isArray(raw) ? raw : String(raw).split(',');
return Array.from(new Set(arr.map(t => String(t || '').trim()).filter(Boolean)));
};
const num = (v) => { const n = Number(v); return Number.isFinite(n) && n > 0 ? Math.floor(n) : 0; };
const parseMedia = (raw) => {
if (!raw) return null;
if (typeof raw === 'object' && raw.blobId) return { kind: String(raw.kind || 'audio'), name: safeText(raw.name), blobId: String(raw.blobId) };
const m = String(raw).match(MEDIA_RE);
if (!m) return null;
return { kind: m[1], name: safeText(m[2]) || 'file', blobId: m[3] };
};
module.exports = ({ cooler }) => {
let ssb;
const openSsb = async () => { if (!ssb) ssb = await cooler.open(); return ssb; };
const publish = (ssbClient, content) => new Promise((resolve, reject) => ssbClient.publish(content, (err, res) => err ? reject(err) : resolve(res)));
const buildIndex = (messages) => {
const tomb = buildValidatedTombstoneSet(messages);
const nodes = new Map();
const parentOf = new Map();
const opinions = [];
const plays = [];
for (const m of messages) {
const v = m.value || {};
const c = v.content;
if (!c || typeof c !== 'object') continue;
if (c.type === CHANNEL_TYPE || c.type === EPISODE_TYPE) {
nodes.set(m.key, { key: m.key, type: c.type, author: v.author, ts: v.timestamp || m.timestamp || 0, c });
if (typeof c.replaces === 'string') parentOf.set(m.key, c.replaces);
} else if (c.type === OPINION_TYPE && typeof c.target === 'string') {
opinions.push({ target: c.target, author: v.author, category: String(c.category || ''), ts: v.timestamp || 0 });
} else if (c.type === PLAY_TYPE && typeof c.target === 'string') {
plays.push({ target: c.target, author: v.author, ts: v.timestamp || 0 });
}
}
const rootOf = (key) => {
const seen = new Set();
let cur = key;
while (parentOf.has(cur) && !seen.has(cur)) { seen.add(cur); cur = parentOf.get(cur); }
return cur;
};
const versionsByRoot = new Map();
for (const node of nodes.values()) {
const root = rootOf(node.key);
if (tomb.has(root)) continue;
if (!versionsByRoot.has(root)) versionsByRoot.set(root, []);
versionsByRoot.get(root).push(node);
}
const tips = new Map();
for (const [root, versions] of versionsByRoot) {
versions.sort((a, b) => a.ts - b.ts);
const first = versions[0];
const own = versions.filter(v => v.author === first.author && v.type === first.type);
tips.set(root, { root, first, tip: own[own.length - 1] });
}
const episodes = new Map();
const opinionsByRoot = new Map();
for (const op of opinions) {
if (!nodes.has(op.target)) continue;
const r = rootOf(op.target);
if (!opinionsByRoot.has(r)) opinionsByRoot.set(r, []);
opinionsByRoot.get(r).push(op);
}
const playsByRoot = new Map();
for (const pl of plays) {
if (!nodes.has(pl.target)) continue;
const r = rootOf(pl.target);
if (!playsByRoot.has(r)) playsByRoot.set(r, new Set());
playsByRoot.get(r).add(pl.author);
}
for (const { root, first, tip } of tips.values()) {
if (first.type !== EPISODE_TYPE) continue;
const c = tip.c;
const channelId = safeText(c.channel);
if (!channelId) continue;
const ops = {};
const voters = [];
for (const op of (opinionsByRoot.get(root) || [])) {
if (voters.includes(op.author) || !categories.includes(op.category)) continue;
voters.push(op.author);
ops[op.category] = (ops[op.category] || 0) + 1;
}
const listeners = Array.from(playsByRoot.get(root) || []);
episodes.set(root, {
id: root, rootId: root, tipId: tip.key, channelId,
number: num(c.number), title: safeText(c.title), description: safeText(c.description),
media: parseMedia(c.media), tags: normalizeTags(c.tags),
author: first.author, createdAt: first.c.createdAt || new Date(first.ts).toISOString(),
updatedAt: c.updatedAt || new Date(tip.ts).toISOString(), ts: tip.ts, publishedTs: Date.parse(first.c.createdAt || '') || first.ts,
opinions: ops, opinions_inhabitants: voters, opinionCount: voters.length,
listeners, playCount: listeners.length
});
}
const channels = new Map();
for (const { root, first, tip } of tips.values()) {
if (first.type !== CHANNEL_TYPE) continue;
const c = tip.c;
const eps = Array.from(episodes.values()).filter(e => e.channelId === root && e.author === first.author).sort((a, b) => a.publishedTs - b.publishedTs || a.number - b.number);
eps.forEach((e, i) => { if (!e.number) e.number = i + 1; });
const playCount = eps.reduce((s, e) => s + e.playCount, 0);
const ops = {};
const voters = [];
for (const op of (opinionsByRoot.get(root) || [])) {
if (voters.includes(op.author) || !categories.includes(op.category)) continue;
voters.push(op.author);
ops[op.category] = (ops[op.category] || 0) + 1;
}
const opinionCount = voters.length + eps.reduce((s, e) => s + e.opinionCount, 0);
channels.set(root, {
id: root, rootId: root, tipId: tip.key,
title: safeText(c.title), description: safeText(c.description), category: normalizeCategory(c.category),
cover: parseMedia(c.cover), tags: normalizeTags(c.tags),
author: first.author, createdAt: first.c.createdAt || new Date(first.ts).toISOString(),
updatedAt: c.updatedAt || new Date(tip.ts).toISOString(), ts: tip.ts,
episodes: eps, episodeCount: eps.length, playCount, opinionCount,
opinions: ops, opinions_inhabitants: voters,
lastEpisodeTs: eps.length ? eps[eps.length - 1].publishedTs : 0,
lastActivityTs: Math.max(tip.ts, ...eps.map(e => e.ts))
});
}
return { channels, episodes, rootOf, nodes };
};
const load = async () => {
const ssbClient = await openSsb();
const idx = memoIndex('podcasts', await readTyped(ssbClient, TYPES, { limit: logLimit }), buildIndex);
return { ssbClient, idx };
};
const findChannel = (idx, id) => {
const key = String(id || '');
if (idx.channels.has(key)) return idx.channels.get(key);
if (idx.nodes.has(key)) return idx.channels.get(idx.rootOf(key)) || null;
return null;
};
const findEpisode = (idx, id) => {
const key = String(id || '');
if (idx.episodes.has(key)) return idx.episodes.get(key);
if (idx.nodes.has(key)) return idx.episodes.get(idx.rootOf(key)) || null;
return null;
};
return {
CATEGORIES,
parseMedia,
async createChannel({ title, description, category, cover, tags }) {
const ssbClient = await openSsb();
const t = safeText(title);
if (!t) throw new Error('Title is required');
const now = new Date().toISOString();
const media = parseMedia(cover);
return publish(ssbClient, {
type: CHANNEL_TYPE, title: t, description: safeText(description), category: normalizeCategory(category),
...(media ? { cover: media } : {}), tags: normalizeTags(tags), author: ssbClient.id, createdAt: now, updatedAt: now
});
},
async updateChannel(id, data = {}) {
const { ssbClient, idx } = await load();
const ch = findChannel(idx, id);
if (!ch) throw new Error('Podcast not found');
if (ch.author !== ssbClient.id) throw new Error('Only the author can update this podcast');
const media = data.cover !== undefined && data.cover ? parseMedia(data.cover) : ch.cover;
const res = await publish(ssbClient, {
type: CHANNEL_TYPE,
title: safeText(data.title) || ch.title,
description: data.description !== undefined ? safeText(data.description) : ch.description,
category: data.category !== undefined ? normalizeCategory(data.category) : ch.category,
...(media ? { cover: media } : {}),
tags: data.tags !== undefined ? normalizeTags(data.tags) : ch.tags,
author: ssbClient.id, createdAt: ch.createdAt, updatedAt: new Date().toISOString(), replaces: ch.tipId
});
return { key: res.key, rootId: ch.id };
},
async deleteChannel(id) {
const { ssbClient, idx } = await load();
const ch = findChannel(idx, id);
if (!ch) throw new Error('Podcast not found');
if (ch.author !== ssbClient.id) throw new Error('Only the author can delete this podcast');
await publish(ssbClient, { type: 'tombstone', target: ch.id, deletedAt: new Date().toISOString(), author: ssbClient.id });
return { key: ch.id };
},
async addEpisode(channelId, { title, description, media, number, tags }) {
const { ssbClient, idx } = await load();
const ch = findChannel(idx, channelId);
if (!ch) throw new Error('Podcast not found');
if (ch.author !== ssbClient.id) throw new Error('Only the author can publish episodes');
const m = parseMedia(media);
if (!m || (m.kind !== 'audio' && m.kind !== 'video')) throw new Error('An audio or video file is required');
const t = safeText(title);
if (!t) throw new Error('Title is required');
const now = new Date().toISOString();
const res = await publish(ssbClient, {
type: EPISODE_TYPE, channel: ch.id, number: num(number) || ch.episodeCount + 1,
title: t, description: safeText(description), media: m, tags: normalizeTags(tags),
author: ssbClient.id, createdAt: now, updatedAt: now
});
return { key: res.key, rootId: res.key, channelId: ch.id };
},
async updateEpisode(id, data = {}) {
const { ssbClient, idx } = await load();
const ep = findEpisode(idx, id);
if (!ep) throw new Error('Episode not found');
if (ep.author !== ssbClient.id) throw new Error('Only the author can update this episode');
const m = data.media ? parseMedia(data.media) : ep.media;
const res = await publish(ssbClient, {
type: EPISODE_TYPE, channel: ep.channelId, number: data.number !== undefined && num(data.number) ? num(data.number) : ep.number,
title: safeText(data.title) || ep.title, description: data.description !== undefined ? safeText(data.description) : ep.description,
media: m, tags: data.tags !== undefined ? normalizeTags(data.tags) : ep.tags,
author: ssbClient.id, createdAt: ep.createdAt, updatedAt: new Date().toISOString(), replaces: ep.tipId
});
return { key: res.key, rootId: ep.id, channelId: ep.channelId };
},
async deleteEpisode(id) {
const { ssbClient, idx } = await load();
const ep = findEpisode(idx, id);
if (!ep) throw new Error('Episode not found');
if (ep.author !== ssbClient.id) throw new Error('Only the author can delete this episode');
await publish(ssbClient, { type: 'tombstone', target: ep.id, deletedAt: new Date().toISOString(), author: ssbClient.id });
return { key: ep.id, channelId: ep.channelId };
},
async markPlayed(id) {
const { ssbClient, idx } = await load();
const ep = findEpisode(idx, id);
if (!ep) throw new Error('Episode not found');
if (ep.listeners.includes(ssbClient.id)) return null;
return publish(ssbClient, { type: PLAY_TYPE, target: ep.id, createdAt: new Date().toISOString() });
},
async createOpinion(id, category) {
if (!categories.includes(category)) throw new Error('Invalid voting category');
const { ssbClient, idx } = await load();
const target = findEpisode(idx, id) || findChannel(idx, id);
if (!target) throw new Error('Episode not found');
if (target.opinions_inhabitants.includes(ssbClient.id)) throw new Error('Already voted');
return publish(ssbClient, { type: OPINION_TYPE, target: target.id, category, createdAt: new Date().toISOString() });
},
async getChannelById(id) {
const { idx } = await load();
return findChannel(idx, id);
},
async getAnyById(id) {
const { idx } = await load();
return findChannel(idx, id) || findEpisode(idx, id);
},
async getEpisodeById(id) {
const { idx } = await load();
const ep = findEpisode(idx, id);
if (!ep) return null;
return { ...ep, channel: idx.channels.get(ep.channelId) || null };
},
async listAll({ filter = 'all', q = '', category = '', viewerId = null } = {}) {
const { ssbClient, idx } = await load();
const me = viewerId || ssbClient.id;
const f = String(filter || 'all').toLowerCase();
const needle = String(q || '').trim().toLowerCase();
const cat = normU(category);
let out = Array.from(idx.channels.values());
if (f === 'mine') out = out.filter(c => c.author === me);
else if (f === 'recent') out = out.filter(c => c.lastActivityTs >= Date.now() - RECENT_MS);
else if (f === 'viewers') out = out.filter(c => c.playCount > 0);
else if (f === 'top') out = out.filter(c => c.opinionCount > 0 || (c.spreadCount || 0) > 0);
else if (CATEGORIES.includes(f.toUpperCase())) out = out.filter(c => c.category === f.toUpperCase());
if (cat && CATEGORIES.includes(cat)) out = out.filter(c => c.category === cat);
if (needle) out = out.filter(c => [c.title, c.description, c.category, ...c.tags, ...c.episodes.map(e => e.title)].some(v => String(v || '').toLowerCase().includes(needle)));
if (f === 'viewers') out.sort((a, b) => b.playCount - a.playCount || b.lastActivityTs - a.lastActivityTs);
else if (f === 'top') out.sort((a, b) => b.opinionCount - a.opinionCount || b.playCount - a.playCount);
else out.sort((a, b) => b.lastActivityTs - a.lastActivityTs);
return out;
},
async listEpisodes({ filter = 'all', q = '' } = {}) {
const { ssbClient, idx } = await load();
const me = ssbClient.id;
const f = String(filter || 'all').toLowerCase();
const needle = String(q || '').trim().toLowerCase();
let out = Array.from(idx.episodes.values()).filter(e => idx.channels.has(e.channelId)).map(e => ({ ...e, channel: idx.channels.get(e.channelId) }));
if (f === 'mine') out = out.filter(e => e.author === me);
else if (f === 'recent') out = out.filter(e => e.ts >= Date.now() - RECENT_MS);
else if (f === 'viewers') out = out.filter(e => e.playCount > 0);
else if (f === 'top') out = out.filter(e => e.opinionCount > 0);
if (needle) out = out.filter(e => [e.title, e.description, ...e.tags, e.channel.title].some(v => String(v || '').toLowerCase().includes(needle)));
if (f === 'viewers') out.sort((a, b) => b.playCount - a.playCount);
else if (f === 'top') out.sort((a, b) => b.opinionCount - a.opinionCount);
else out.sort((a, b) => b.publishedTs - a.publishedTs);
return out;
},
async resolveRootId(id) {
const { idx } = await load();
const ch = findChannel(idx, id);
if (ch) return ch.id;
const ep = findEpisode(idx, id);
return ep ? ep.id : String(id || '');
}
};
};

104
src/models/polls_model.js
View file

@ -1,6 +1,8 @@
const pull = require('../server/node_modules/pull-stream');
const { getConfig } = require('../configs/config-manager.js');
const { buildValidatedTombstoneSet } = require('./tombstone_validator');
const { readTyped, memoIndex } = require('./typed_log');
const { versionGroups } = require('../backend/dedupe');
const logLimit = getConfig().ssbLogStream?.limit || 1000;
@ -37,13 +39,9 @@ module.exports = ({ cooler, isPublic = false, tribeCrypto = null, chatsModel = n
let ssb;
const openSsb = async () => { if (!ssb) ssb = await cooler.open(); return ssb; };
const getAllMessages = async (ssbClient) =>
new Promise((resolve, reject) => {
pull(
ssbClient.createLogStream({ limit: logLimit }),
pull.collect((err, msgs) => (err ? reject(err) : resolve(msgs)))
);
});
const POLLS_TYPES = [POLL_TYPE, VOTE_TYPE, OPINION_TYPE, 'pollClose', 'post', 'tombstone'];
const getAllMessages = async (ssbClient) => readTyped(ssbClient, POLLS_TYPES, { limit: logLimit });
const decryptField = (value, keys) => {
if (!tribeCrypto || typeof value !== 'string' || !value) return null;
@ -65,82 +63,61 @@ module.exports = ({ cooler, isPublic = false, tribeCrypto = null, chatsModel = n
const buildIndex = (messages) => {
const tomb = buildValidatedTombstoneSet(messages);
const nodes = new Map();
const replaced = new Map();
const votesByRoot = new Map();
const opinionsByRoot = new Map();
const closedRoots = new Set();
const commentsByRoot = new Map();
const authorByKey = new Map();
const others = [];
for (const m of messages) {
const v = m && m.value;
const c = v && v.content;
if (!c || typeof c !== 'object') continue;
authorByKey.set(m.key, v.author);
if (c.type === POLL_TYPE) nodes.set(m.key, { key: m.key, author: v.author, seq: v.sequence || 0, ts: v.timestamp || m.timestamp || 0, c });
else others.push(m);
}
if (c.type === POLL_TYPE) {
nodes.set(m.key, { key: m.key, author: v.author, ts: v.timestamp || m.timestamp || 0, c });
if (typeof c.replaces === 'string') replaced.set(c.replaces, m.key);
continue;
}
const vg = versionGroups(Array.from(nodes.values()).map(n => ({ key: n.key, author: n.author, seq: n.seq, ts: n.ts, replaces: typeof n.c.replaces === 'string' ? n.c.replaces : null, claimedAuthor: typeof n.c.author === 'string' ? n.c.author : null })));
const rootOf = vg.rootOf;
const tipOf = vg.tipOf;
for (const m of others) {
const v = m.value;
const c = v.content;
if (c.type === VOTE_TYPE && typeof c.target === 'string') {
const entry = votesByRoot.get(c.target) || new Map();
const prev = entry.get(v.author);
const ts = v.timestamp || m.timestamp || 0;
if (!prev || ts >= prev.ts) {
entry.set(v.author, {
ts,
choices: Array.isArray(c.choices) ? c.choices : [],
encryptedChoices: typeof c.encryptedChoices === 'string' ? c.encryptedChoices : null
});
}
votesByRoot.set(c.target, entry);
const target = rootOf(c.target);
const entry = votesByRoot.get(target) || new Map();
const list = entry.get(v.author) || [];
list.push({
ts: v.timestamp || m.timestamp || 0,
choices: Array.isArray(c.choices) ? c.choices : [],
encryptedChoices: typeof c.encryptedChoices === 'string' ? c.encryptedChoices : null
});
entry.set(v.author, list);
votesByRoot.set(target, entry);
continue;
}
if (c.type === OPINION_TYPE && typeof c.target === 'string') {
const entry = opinionsByRoot.get(c.target) || { counts: {}, voters: [] };
const target = rootOf(c.target);
const entry = opinionsByRoot.get(target) || { counts: {}, voters: [] };
if (!entry.voters.includes(v.author)) {
entry.voters.push(v.author);
entry.counts[c.category] = (entry.counts[c.category] || 0) + 1;
}
opinionsByRoot.set(c.target, entry);
opinionsByRoot.set(target, entry);
continue;
}
if (c.type === 'pollClose' && typeof c.target === 'string') {
closedRoots.add(`${v.author}|${c.target}`);
closedRoots.add(`${v.author}|${rootOf(c.target)}`);
continue;
}
if (c.type === 'post' && typeof c.root === 'string') {
commentsByRoot.set(c.root, (commentsByRoot.get(c.root) || 0) + 1);
const target = rootOf(c.root);
commentsByRoot.set(target, (commentsByRoot.get(target) || 0) + 1);
}
}
const rootOf = (key) => {
let cur = key;
const seen = new Set();
while (nodes.has(cur) && nodes.get(cur).c.replaces && !seen.has(cur)) {
seen.add(cur);
const next = nodes.get(cur).c.replaces;
if (!nodes.has(next)) break;
cur = next;
}
return cur;
};
const tipOf = (root) => {
let cur = root;
const seen = new Set();
while (replaced.has(cur) && !seen.has(cur)) {
seen.add(cur);
const next = replaced.get(cur);
const node = nodes.get(next);
if (!node || node.author !== nodes.get(cur).author) break;
cur = next;
}
return cur;
};
return { tomb, nodes, replaced, votesByRoot, opinionsByRoot, closedRoots, commentsByRoot, rootOf, tipOf };
return { tomb, nodes, vg, votesByRoot, opinionsByRoot, closedRoots, commentsByRoot, rootOf, tipOf };
};
const isExpired = (c) => {
@ -165,7 +142,13 @@ module.exports = ({ cooler, isPublic = false, tribeCrypto = null, chatsModel = n
try { rawOptions = o ? JSON.parse(o) : []; } catch (_) { rawOptions = []; }
}
const options = normalizeOptions(rawOptions);
const voteMap = idx.votesByRoot.get(rootId) || new Map();
const opensAt = (idx.nodes.get(rootId) || {}).ts || 0;
const closesAt = c.deadline ? Date.parse(c.deadline) : NaN;
const voteMap = new Map();
for (const [author, list] of (idx.votesByRoot.get(rootId) || new Map()).entries()) {
const inWindow = list.filter(b => b.ts >= opensAt && (!Number.isFinite(closesAt) || b.ts <= closesAt));
if (inWindow.length) voteMap.set(author, inWindow.reduce((a, b) => (b.ts >= a.ts ? b : a)));
}
const counts = {};
for (const o of options) counts[o] = 0;
@ -223,9 +206,7 @@ module.exports = ({ cooler, isPublic = false, tribeCrypto = null, chatsModel = n
const collect = async (viewerId) => {
const ssbClient = await openSsb();
const messages = await getAllMessages(ssbClient);
const idx = buildIndex(messages);
const roots = new Set();
for (const key of idx.nodes.keys()) roots.add(idx.rootOf(key));
const idx = memoIndex('polls', messages, buildIndex);
const keyCache = new Map();
const keysFor = async (c) => {
@ -239,7 +220,7 @@ module.exports = ({ cooler, isPublic = false, tribeCrypto = null, chatsModel = n
};
const list = [];
for (const rootId of roots) {
for (const rootId of idx.vg.roots) {
const tipId = idx.tipOf(rootId);
if (idx.tomb.has(tipId) || idx.tomb.has(rootId)) continue;
const node = idx.nodes.get(tipId);
@ -275,6 +256,7 @@ module.exports = ({ cooler, isPublic = false, tribeCrypto = null, chatsModel = n
else if (f === 'OPEN') out = out.filter(p => p.status === 'OPEN');
else if (f === 'CLOSED') out = out.filter(p => p.status === 'CLOSED');
else if (f === 'VOTED') out = out.filter(p => p.hasVoted);
else if (f === 'RECENT') out = out.filter(p => (Date.parse(p.createdAt || '') || 0) >= Date.now() - 86400000);
else if (f === 'FAVORITES') {
const fav = new Set((opts.favorites || []).map(String));
out = out.filter(p => fav.has(String(p.id)));

10
src/models/projects_model.js
View file

@ -3,6 +3,7 @@ const moment = require("../server/node_modules/moment")
const { getConfig } = require("../configs/config-manager.js")
const { buildValidatedTombstoneSet } = require('./tombstone_validator');
const { dedupeBy, norm } = require('../backend/dedupe')
const { readTyped } = require("./typed_log")
const logLimit = (getConfig().ssbLogStream && getConfig().ssbLogStream.limit) || 1000
module.exports = ({ cooler }) => {
@ -20,13 +21,10 @@ module.exports = ({ cooler }) => {
return Math.max(0, Math.min(100, x))
}
const PROJECT_TYPES = ["project", "projectOpinion", "projectFollow", "projectPledge", "projectClaim", "projectPledgeConfirm", "tombstone"]
async function getAllMsgs(ssbClient) {
return new Promise((r, j) => {
pull(
ssbClient.createLogStream({ limit: logLimit }),
pull.collect((e, m) => (e ? j(e) : r(m)))
)
})
return readTyped(ssbClient, PROJECT_TYPES, { limit: logLimit })
}
function extractBlobId(possibleMarkdownImage) {

93
src/models/pub_invites.js Normal file
View file

@ -0,0 +1,93 @@
const ssbKeys = require('../server/node_modules/ssb-keys');
const { readTyped } = require('./typed_log');
const ANNOUNCE_TYPE = 'pub-invites';
const MAX_CODES = 40;
const FEED_ID = /@[A-Za-z0-9+/]{43}=\.ed25519/;
const SHS_KEY = /~shs:([A-Za-z0-9+/]{43}=)/;
const LEGACY_ADDRESS = /^([^:~\s]+):(\d+):@/;
const MODERN_ADDRESS = /^(?:net|onion):([^:~\s]+):(\d+)~/;
const parseInvite = (code) => {
const s = String(code || '').trim();
const legacy = s.match(FEED_ID);
const modern = s.match(SHS_KEY);
const pub = legacy ? legacy[0] : modern ? `@${modern[1]}.ed25519` : null;
const address = s.match(LEGACY_ADDRESS) || s.match(MODERN_ADDRESS);
const host = address ? address[1] : null;
const port = address ? Number(address[2]) : null;
let id = null;
try {
const seed = Buffer.from(s.split('~').pop().replace(/^invite:/, ''), 'base64');
if (seed.length === 32) id = ssbKeys.generate('ed25519', seed).id;
} catch (_) {}
return { pub, id, host, port };
};
const count = (n) => Math.max(0, Math.floor(Number(n) || 0));
const summarize = (rows) => {
const list = (Array.isArray(rows) ? rows : [])
.filter((r) => r && typeof r.id === 'string' && count(r.total) > 0)
.map((r) => ({ id: r.id, total: count(r.total), used: Math.min(count(r.used), count(r.total)) }));
const codes = list.filter((r) => r.total > 1)
.sort((a, b) => (b.total - b.used) - (a.total - a.used))
.slice(0, MAX_CODES)
.sort((a, b) => (a.id < b.id ? -1 : a.id > b.id ? 1 : 0));
return {
total: list.reduce((n, r) => n + r.total, 0),
used: list.reduce((n, r) => n + r.used, 0),
codes
};
};
const latestByAuthor = (msgs) => {
const out = new Map();
for (const m of msgs || []) {
const v = m && m.value;
const c = v && v.content;
if (!c || c.type !== ANNOUNCE_TYPE) continue;
const prev = out.get(v.author);
if (!prev || (Number(v.sequence) || 0) > (Number(prev.value.sequence) || 0)) out.set(v.author, m);
}
return out;
};
const listUses = (ssb) => new Promise((resolve) => {
if (!ssb || !ssb.invite || typeof ssb.invite.list !== 'function') return resolve(null);
try { ssb.invite.list((err, rows) => resolve(err ? null : rows)); } catch (_) { resolve(null); }
});
const announce = async (ssb) => {
const rows = await listUses(ssb);
if (!rows) return null;
const next = summarize(rows);
const last = latestByAuthor(await readTyped(ssb, [ANNOUNCE_TYPE])).get(ssb.id);
const c = last && last.value.content;
const prev = c ? { total: c.total, used: c.used, codes: c.codes } : null;
if (prev ? JSON.stringify(prev) === JSON.stringify(next) : !next.total) return null;
return new Promise((resolve, reject) => ssb.publish({ type: ANNOUNCE_TYPE, ...next, updatedAt: new Date().toISOString() }, (err, msg) => (err ? reject(err) : resolve(msg))));
};
const statusFor = (entries, msgs) => {
const latest = latestByAuthor(msgs);
return (Array.isArray(entries) ? entries : []).map((e) => {
const { pub, id } = parseInvite(e && e.code);
const m = pub ? latest.get(pub) : null;
if (!m) return { ...e, invites: null, exhausted: false };
const c = m.value.content;
const total = count(c.total);
const used = Math.min(count(c.used), total);
const own = id && Array.isArray(c.codes) ? c.codes.find((x) => x && x.id === id) : null;
const exhausted = (total > 0 && used >= total) || (!!own && count(own.used) >= count(own.total));
return { ...e, invites: total > 0 ? { used, total } : null, exhausted };
});
};
const readStatus = async (ssb, entries) => {
let msgs = [];
try { msgs = await readTyped(ssb, [ANNOUNCE_TYPE]); } catch (_) { msgs = []; }
return statusFor(entries, msgs);
};
module.exports = { ANNOUNCE_TYPE, parseInvite, summarize, announce, readStatus };

57
src/models/reports_model.js
View file

@ -2,7 +2,8 @@ const pull = require('../server/node_modules/pull-stream');
const { normalizeImages, normalizeVideo } = require('./media_gallery');
const { getConfig } = require('../configs/config-manager.js');
const { buildValidatedTombstoneSet } = require('./tombstone_validator');
const { dedupeBy, norm } = require('../backend/dedupe');
const { dedupeBy, norm, versionGroups } = require('../backend/dedupe');
const { readTyped, memoIndex } = require('./typed_log');
const logLimit = getConfig().ssbLogStream?.limit || 1000;
const normU = (v) => String(v || '').trim().toUpperCase();
@ -50,20 +51,13 @@ module.exports = ({ cooler }) => {
let ssb;
const openSsb = async () => { if (!ssb) ssb = await cooler.open(); return ssb; };
const getAllMessages = async (ssbClient) =>
new Promise((resolve, reject) => {
pull(
ssbClient.createLogStream({ limit: logLimit }),
pull.collect((err, msgs) => err ? reject(err) : resolve(msgs))
);
});
const REPORT_TYPES = ['report', 'reportConfirm', 'reportOpinion', 'tombstone'];
const getAllMessages = async (ssbClient) => readTyped(ssbClient, REPORT_TYPES, { limit: logLimit });
const buildIndex = (messages) => {
const tomb = buildValidatedTombstoneSet(messages);
const nodes = new Map();
const parent = new Map();
const child = new Map();
const strictChild = new Map();
const confirms = [];
const opinionMsgs = [];
@ -79,46 +73,24 @@ module.exports = ({ cooler }) => {
if (c.type !== 'report') continue;
const ts = v.timestamp || m.timestamp || 0;
nodes.set(k, { key: k, ts, c, author: v.author });
if (c.replaces) {
parent.set(k, c.replaces);
child.set(c.replaces, k);
}
nodes.set(k, { key: k, ts, seq: v.sequence || 0, c, author: v.author });
}
for (const [k, node] of nodes) {
const t = node.c.replaces;
if (t) { const orig = nodes.get(t); if (orig && orig.author === node.author) strictChild.set(t, k); }
}
const rootOf = (id) => {
let cur = id, g = 0;
while (parent.has(cur) && nodes.has(parent.get(cur)) && g++ < 100000) cur = parent.get(cur);
return cur;
};
const contentTipOf = (root) => {
let cur = root, g = 0;
while (strictChild.has(cur) && g++ < 100000) cur = strictChild.get(cur);
const n = nodes.get(cur), rn = nodes.get(root);
return (n && rn && n.author === rn.author) ? cur : root;
};
const roots = new Set();
for (const id of nodes.keys()) roots.add(rootOf(id));
const vg = versionGroups([...nodes.values()].map(n => ({ key: n.key, author: n.author, seq: n.seq, ts: n.ts, replaces: n.c.replaces || null, claimedAuthor: typeof n.c.author === 'string' ? n.c.author : null })));
const rootOf = vg.rootOf;
const contentTipOf = vg.tipOf;
const confirmsByRoot = new Map(), opinionsByRoot = new Map();
for (const cf of confirms) { if (!nodes.has(cf.target)) continue; const r = rootOf(cf.target); if (!confirmsByRoot.has(r)) confirmsByRoot.set(r, []); confirmsByRoot.get(r).push(cf); }
for (const op of opinionMsgs) { if (!nodes.has(op.target)) continue; const r = rootOf(op.target); if (!opinionsByRoot.has(r)) opinionsByRoot.set(r, []); opinionsByRoot.get(r).push(op); }
const tipByRoot = new Map();
for (const r of roots) tipByRoot.set(r, contentTipOf(r));
for (const r of vg.roots) tipByRoot.set(r, contentTipOf(r));
const resolveGroup = (root) => {
const contentTip = contentTipOf(root);
const contentNode = nodes.get(contentTip) || nodes.get(root);
if (!contentNode) return null;
if (!contentNode || !tipByRoot.has(root)) return null;
const rootAuthor = nodes.get(root) ? nodes.get(root).author : null;
const lc = contentNode.c || {};
@ -126,8 +98,7 @@ module.exports = ({ cooler }) => {
const voterSet = new Set(ensureArray(lc.opinions_inhabitants));
const confirmSet = new Set(ensureArray(lc.confirmations));
const groupKeys = []; { let x = root, g = 0; groupKeys.push(x); while (child.has(x) && g++ < 100000) { x = child.get(x); groupKeys.push(x); } }
for (const k of groupKeys) {
for (const k of (vg.groups.get(root) || [root])) {
const n = nodes.get(k); if (!n) continue; if (n.author !== rootAuthor) continue; const c = n.c || {};
for (const cb of ensureArray(c.confirmations)) confirmSet.add(cb);
for (const vv of ensureArray(c.opinions_inhabitants)) voterSet.add(vv);
@ -151,7 +122,7 @@ module.exports = ({ cooler }) => {
};
};
return { tomb, nodes, parent, child, rootOf, tipByRoot, resolveGroup };
return { tomb, nodes, rootOf, tipByRoot, resolveGroup };
};
const buildReport = (node, agg) => {
@ -304,7 +275,7 @@ module.exports = ({ cooler }) => {
const ssb = await openSsb();
const messages = await getAllMessages(ssb);
const idx = buildIndex(messages);
const idx = memoIndex('reports', messages, buildIndex);
const root = idx.rootOf(id);
const agg = idx.resolveGroup(root);
if (!agg || agg.tombstoned) throw new Error('Report not found');

821
src/models/rooms_model.js Normal file
View file

@ -0,0 +1,821 @@
const crypto = require("crypto")
const { buildValidatedTombstoneSet } = require('./tombstone_validator')
const { collabContent, openInviteOf } = require('../backend/collab_content')
const { readTyped, memoIndex } = require('./typed_log')
const { roomNumberOf, normalizeNumber } = require('./phone_number')
const { getConfig } = require("../configs/config-manager.js")
const roomCollab = collabContent({ membersField: 'members', undecField: 'undecryptable', contentFields: ['title', 'description', 'image', 'status'], listFields: ['tags', 'invites'] })
const logLimit = getConfig().ssbLogStream?.limit || 1000
const ROOM_MAX = 50
const ROOM_TYPES = ["room", "roomMember", "tribe-keys", "tombstone"]
const STATUSES = ["OPEN", "INVITE-ONLY"]
const INVITE_SALT = "SolarNET.HuB-rooms"
const INVITE_BYTES = 16
const RECENT_MS = 86400000
const RID = /^[0-9a-f]{32}$/
const FEED_ID = /^@[A-Za-z0-9+/]{43}=\.ed25519$/
const BLOB_ID = /^&[A-Za-z0-9+/]{43}=\.sha256$/
const safeText = (v) => String(v || "").trim()
const normalizeTags = (raw) => {
if (!raw) return []
if (Array.isArray(raw)) return raw.map(t => String(t || "").trim()).filter(Boolean)
return String(raw).split(",").map(t => t.trim()).filter(Boolean)
}
const normalizeStatus = (v) => (STATUSES.includes(String(v || "").toUpperCase()) ? String(v).toUpperCase() : "OPEN")
const blobRef = (v) => (BLOB_ID.test(safeText(v)) ? safeText(v) : "")
const asP = (fn, ...args) => new Promise((resolve, reject) => fn(...args, (err, v) => err ? reject(err) : resolve(v)))
const rpcValue = (fn, ...args) => new Promise((resolve) => {
try {
const r = fn(...args, (err, v) => resolve(err ? null : v))
if (r && typeof r.then === "function") r.then(resolve, () => resolve(null))
else if (r !== undefined) resolve(r)
} catch (_) { resolve(null) }
})
module.exports = ({ cooler, tribeCrypto, roomCrypto, tribesModel }) => {
let ssb
const openSsb = async () => { if (!ssb) ssb = await cooler.open(); return ssb }
const phone = async () => { const s = await openSsb(); return s && s.phone ? s.phone : null }
const ownCrypto = roomCrypto || tribeCrypto
const lookupKey = (rid) => (ownCrypto && ownCrypto.getKey(rid)) || (tribeCrypto && tribeCrypto.getKey(rid)) || null
const lookupKeys = (rid) => {
const a = (ownCrypto && ownCrypto.getKeys(rid)) || []
if (a.length) return a
return (tribeCrypto && tribeCrypto.getKeys(rid)) || []
}
const lookupGen = (rid) => ((ownCrypto && ownCrypto.getGen(rid)) || (tribeCrypto && tribeCrypto.getGen(rid)) || 0)
const setRoomKey = (rootId, keyHex) => { if (ownCrypto) ownCrypto.setKey(rootId, keyHex, 1) }
const hashCode = (code, salt) => (ownCrypto && typeof ownCrypto.hashInviteCode === "function") ? ownCrypto.hashInviteCode(code, salt) : (tribeCrypto ? tribeCrypto.hashInviteCode(code, salt) : null)
const inviteMatches = (inv, token) => {
if (!token) return false
if (typeof inv === "string") return inv === token
if (!inv || typeof inv !== "object") return false
if (typeof inv.code === "string" && inv.code === token) return true
if (typeof inv.ch === "string" && (inv.ch === token || hashCode(token, inv.salt) === inv.ch)) return true
return false
}
const inviteToken = (inv) => (inv && typeof inv === "object" && typeof inv.ch === "string") ? inv.ch : (typeof inv === "string" ? inv : (inv && inv.code) || "")
const keyProofFor = (keyHex, id) => { try { return crypto.createHmac("sha256", Buffer.from(keyHex, "hex")).update(String(id), "utf8").digest("hex") } catch (_) { return null } }
const deriveRid = (keyHex, roomId) => crypto.createHmac("sha256", Buffer.from(String(keyHex), "hex")).update("oasis-room-rid|" + roomId).digest("hex").slice(0, 32)
const currentKeyFor = (rootId, tribeId, tribeKeys) => (tribeId && Array.isArray(tribeKeys) && tribeKeys.length ? tribeKeys[0] : lookupKeys(rootId)[0]) || null
const encryptField = (text, keyHex) => {
const key = Buffer.from(keyHex, "hex")
const iv = crypto.randomBytes(12)
const cipher = crypto.createCipheriv("aes-256-gcm", key, iv)
const enc = Buffer.concat([cipher.update(text, "utf8"), cipher.final()])
const authTag = cipher.getAuthTag()
return iv.toString("hex") + authTag.toString("hex") + enc.toString("hex")
}
const tryDecryptField = (encrypted, keyHex) => {
const key = Buffer.from(keyHex, "hex")
const iv = Buffer.from(encrypted.slice(0, 24), "hex")
const authTag = Buffer.from(encrypted.slice(24, 56), "hex")
const ciphertext = Buffer.from(encrypted.slice(56), "hex")
const decipher = crypto.createDecipheriv("aes-256-gcm", key, iv)
decipher.setAuthTag(authTag)
return Buffer.concat([decipher.update(ciphertext), decipher.final()]).toString("utf8")
}
const decryptField = (encrypted, keyHex) => {
try { return tryDecryptField(encrypted, keyHex) } catch (_) { return "" }
}
const getTribeKeysFor = async (tribeId) => {
if (!tribeCrypto || !tribesModel || !tribeId) return []
try {
const rootId = await tribesModel.getRootId(tribeId)
return tribeCrypto.getKeys(rootId) || []
} catch (_) { return [] }
}
const fieldsWith = (c, keyHex, strict) => {
const dec = strict ? tryDecryptField : decryptField
const title = c.title ? dec(c.title, keyHex) : ""
const opt = (v) => { try { return v ? dec(v, keyHex) : "" } catch (_) { return "" } }
return {
title: safeText(title),
description: safeText(opt(c.description)),
image: blobRef(opt(c.image)),
tags: normalizeTags(opt(c.tags)),
rid: RID.test(opt(c.rid)) ? opt(c.rid) : ""
}
}
const decryptWithKeys = (c, keys) => {
if (!c.title || !keys.length) return null
for (const k of keys) {
try { return fieldsWith(c, k, true) } catch (_) {}
}
return null
}
const encryptForInvite = (roomKeyHex, code, saltHex) => {
const salt = saltHex ? Buffer.from(saltHex, "hex") : Buffer.from(INVITE_SALT)
const derived = crypto.scryptSync(code, salt, 32)
return encryptField(roomKeyHex, derived.toString("hex"))
}
const decryptFromInvite = (encryptedKey, code, saltHex) => {
const salt = saltHex ? Buffer.from(saltHex, "hex") : Buffer.from(INVITE_SALT)
const derived = crypto.scryptSync(code, salt, 32)
return decryptField(encryptedKey, derived.toString("hex"))
}
const tryDecryptPublicInviteKey = (invites) => {
if (!Array.isArray(invites)) return null
for (const inv of invites) {
if (!inv || typeof inv !== "object" || inv.public !== true) continue
if (typeof inv.code !== "string" || typeof inv.ek !== "string") continue
try {
const key = decryptFromInvite(inv.ek, inv.code, inv.salt)
if (key) return key
} catch (_) {}
}
return null
}
const generateInviteSalt = () => crypto.randomBytes(16).toString("hex")
const rotateRoomKey = async (rootId, remainingMembers) => {
if (!ownCrypto || !tribeCrypto || !rootId) return
if (!lookupKey(rootId)) return
const newKey = crypto.randomBytes(32).toString("hex")
const newGen = ownCrypto.addNewKey(rootId, newKey)
const ssbClient = await openSsb()
const ssbKeys = require("../server/node_modules/ssb-keys")
const memberKeys = {}
for (const m of new Set([ssbClient.id, ...(Array.isArray(remainingMembers) ? remainingMembers : [])])) {
try { memberKeys[m] = tribeCrypto.boxKeyForMember(newKey, m, ssbKeys) } catch (_) {}
}
if (Object.keys(memberKeys).length) {
await new Promise((resolve) => {
ssbClient.publish({ type: "tribe-keys", tribeId: rootId, generation: newGen, memberKeys }, () => resolve())
})
}
}
const readAll = async (ssbClient) => readTyped(ssbClient, ROOM_TYPES, { limit: logLimit })
const ensureMemberKeys = async (ssbClient, messages, rooms) => {
if (!tribeCrypto || !ownCrypto) return
const leavesByRoot = new Map()
for (const m of messages) {
const c = m.value && m.value.content
if (!c || c.type !== "roomMember" || c.on !== false || !c.target || !c.member) continue
if (!leavesByRoot.has(c.target)) leavesByRoot.set(c.target, [])
leavesByRoot.get(c.target).push({ member: c.member, author: m.value.author, ts: Number(m.timestamp || m.value.timestamp || 0) })
}
const ssbKeys = require("../server/node_modules/ssb-keys")
for (const room of (Array.isArray(rooms) ? rooms : [])) {
if (!room || room.tribeId || room.author !== ssbClient.id) continue
const rootId = room.rootId
if (!rootId || !lookupKey(rootId)) continue
const members = Array.isArray(room.members) ? room.members : []
const leaves = (leavesByRoot.get(rootId) || []).filter(l => l.author === l.member || l.author === room.author)
const plan = ownCrypto.keyPlan({ rootId, ownerId: room.author, gen: lookupGen(rootId), members, messages, leaves })
if (plan.rotate) {
await rotateRoomKey(rootId, members)
continue
}
if (!plan.missing.length) continue
const key = lookupKey(rootId)
const memberKeys = {}
for (const m of plan.missing) {
try { memberKeys[m] = tribeCrypto.boxKeyForMember(key, m, ssbKeys) } catch (_) {}
}
if (!Object.keys(memberKeys).length) continue
await asP(ssbClient.publish, { type: "tribe-keys", tribeId: rootId, generation: lookupGen(rootId) || 1, memberKeys })
}
}
const ingestOwnTribeKeys = async () => {
if (!ownCrypto) return
try {
const ssbClient = await openSsb()
const ssbKeys = require("../server/node_modules/ssb-keys")
const config = require("../server/ssb_config")
const rooms = new Set()
const msgs = await readAll(ssbClient)
for (const m of msgs) {
const c = m.value && m.value.content
if (c && c.type === "room") rooms.add(m.key)
}
const idx = buildIndex(msgs)
const trusted = (root, author) => { const owner = idx.nodes.get(root) && idx.nodes.get(root).author; return !!author && !!owner && (author === owner || author === ssbClient.id) }
for (const m of msgs) {
const c = m.value && m.value.content
if (!c || c.type !== "tribe-keys" || !rooms.has(c.tribeId)) continue
if (!trusted(c.tribeId, m.value.author)) continue
const boxed = c.memberKeys && typeof c.memberKeys === "object" ? c.memberKeys[ssbClient.id] : null
if (!boxed) continue
try {
const unboxed = ssbKeys.unbox(boxed, config.keys)
const key = typeof unboxed === "string" ? unboxed : (unboxed && unboxed.toString ? unboxed.toString() : null)
if (key) ownCrypto.addNewKey(c.tribeId, key)
} catch (_) {}
}
} catch (_) {}
}
const buildIndex = (messages) => {
const tomb = new Set()
const nodes = new Map()
const parent = new Map()
const child = new Map()
const authorByKey = new Map()
const tombRequests = []
const memberMsgs = []
for (const m of messages) {
const k = m.key
const v = m.value || {}
const c = v.content
if (!c) continue
if (c.type === "tombstone" && c.target) { tombRequests.push({ target: c.target, author: v.author }); continue }
if (c.type === "roomMember" && c.target) { memberMsgs.push({ target: c.target, member: c.member, on: c.on !== false, author: v.author, ts: v.timestamp || m.timestamp || 0, code: typeof c.code === "string" ? c.code : "", keyProof: typeof c.keyProof === "string" ? c.keyProof : "" }); continue }
if (c.type === "room") {
nodes.set(k, { key: k, ts: v.timestamp || m.timestamp || 0, seq: v.sequence || 0, c, author: v.author })
authorByKey.set(k, v.author)
if (c.replaces) { parent.set(k, c.replaces); child.set(c.replaces, k) }
}
}
for (const t of tombRequests) {
const targetAuthor = authorByKey.get(t.target)
if (targetAuthor && t.author === targetAuthor) tomb.add(t.target)
}
const rootOf = (id) => { let cur = id, g = 0; while (parent.has(cur) && g++ < 100000) cur = parent.get(cur); return cur }
const rootMemo = new Map()
const groupRootOf = (id) => {
if (rootMemo.has(id)) return rootMemo.get(id)
let cur = id, g = 0
while (parent.has(cur) && nodes.has(parent.get(cur)) && g++ < 100000) cur = parent.get(cur)
rootMemo.set(id, cur)
return cur
}
const ownerTip = new Map()
for (const [k, n] of nodes) {
const r = groupRootOf(k)
const rn = nodes.get(r)
if (!rn || n.author !== rn.author) continue
const prev = ownerTip.has(r) ? nodes.get(ownerTip.get(r)) : null
if (!prev || n.seq > prev.seq || (n.seq === prev.seq && n.ts >= prev.ts)) ownerTip.set(r, k)
}
const contentTipOf = (root) => ownerTip.get(root) || root
const roots = new Set()
for (const id of nodes.keys()) roots.add(groupRootOf(id))
const invitesOf = (root) => {
const out = []
for (const id of [root, contentTipOf(root)]) { const n = nodes.get(id); if (n && Array.isArray(n.c.invites)) out.push(...n.c.invites) }
return out
}
const selfJoinAllowed = (root, mm) => {
const oc = (nodes.get(root) || {}).c || {}
if (oc.encrypted !== true || oc.tribeId) return true
const invites = invitesOf(root)
if (invites.some(inv => inv && typeof inv === "object" && inv.public === true)) return true
if (mm.keyProof && lookupKeys(root).some(k => keyProofFor(k, mm.member) === mm.keyProof)) return true
return !!mm.code && invites.some(inv => inviteMatches(inv, mm.code))
}
const memberByRoot = new Map()
const consumedByRoot = new Map()
for (const mm of memberMsgs) {
if (!nodes.has(mm.target)) continue
const r = groupRootOf(mm.target)
const ownerAuthor = nodes.get(r) && nodes.get(r).author
if (!ownerAuthor || !mm.member) continue
const self = mm.member === mm.author
if (!self && mm.author !== ownerAuthor) continue
if (self && mm.author !== ownerAuthor && mm.on && !selfJoinAllowed(r, mm)) continue
if (!memberByRoot.has(r)) memberByRoot.set(r, new Map())
const m2 = memberByRoot.get(r)
const p = m2.get(mm.member)
if (!p || mm.ts >= p.ts) m2.set(mm.member, { on: mm.on, ts: mm.ts })
if (mm.on && mm.code) {
if (!consumedByRoot.has(r)) consumedByRoot.set(r, new Set())
consumedByRoot.get(r).add(mm.code)
}
}
const isCodeConsumed = (root, code) => {
if (!code) return false
const set = consumedByRoot.get(root)
return !!(set && set.has(code))
}
for (const [k, n] of nodes) {
const r = groupRootOf(k)
const rn = nodes.get(r)
if (!rn || n.author === rn.author || !Array.isArray(n.c.members) || !selfJoinAllowed(r, { member: n.author })) continue
if (!memberByRoot.has(r)) memberByRoot.set(r, new Map())
const m2 = memberByRoot.get(r)
const p = m2.get(n.author)
if (!p || n.ts >= p.ts) m2.set(n.author, { on: n.c.members.includes(n.author), ts: n.ts })
}
const resolveMembers = (root) => {
const ownerNode = nodes.get(root)
const oc = ownerNode ? ownerNode.c : {}
const set = new Set(Array.isArray(oc.members) ? oc.members.filter(x => typeof x === "string" && x) : [])
for (const [mem, st] of (memberByRoot.get(root) || new Map())) { if (st.on) set.add(mem); else set.delete(mem) }
return [...set]
}
const tipByRoot = new Map()
for (const r of roots) tipByRoot.set(r, contentTipOf(r))
return { tomb, nodes, parent, child, rootOf, groupRootOf, contentTipOf, tipByRoot, resolveMembers, isCodeConsumed }
}
const decryptRoomFields = (c, rootId, tribeKeys) => {
if (c.encrypted !== true) {
return { title: safeText(c.title), description: safeText(c.description), image: blobRef(c.image), tags: normalizeTags(c.tags), rid: RID.test(String(c.rid || "")) ? String(c.rid) : "", _undec: false }
}
if (c.tribeId && Array.isArray(tribeKeys) && tribeKeys.length) {
const viaTribe = decryptWithKeys(c, tribeKeys)
if (viaTribe) return { ...viaTribe, _undec: false }
}
for (const k of [...lookupKeys(rootId), tryDecryptPublicInviteKey(c.invites)].filter(Boolean)) {
const out = decryptWithKeys(c, [k])
if (out) return { ...out, _undec: false }
}
return { title: "", description: "", image: "", tags: [], rid: "", _undec: true }
}
const buildRoom = (node, rootId, tribeKeys, members) => {
const c = node.c || {}
if (c.type !== "room") return null
const f = decryptRoomFields(c, rootId, tribeKeys)
const author = c.author || node.author
const roomId = RID.test(String(c.roomId || "")) ? String(c.roomId) : ""
const liveKey = roomId && c.encrypted === true && !f._undec ? currentKeyFor(rootId, c.tribeId, tribeKeys) : null
return {
key: node.key,
rootId,
roomId,
title: f.title,
description: f.description,
image: f.image,
tags: f.tags,
status: ["OPEN", "INVITE-ONLY", "CLOSED"].includes(c.status) ? c.status : "OPEN",
type: c.status === "INVITE-ONLY" || (c.status === "CLOSED" && c.encrypted === true && !c.tribeId) ? "INVITE-ONLY" : "OPEN",
author,
members: Array.isArray(members) ? members : (Array.isArray(c.members) ? c.members : []),
invites: Array.isArray(c.invites) ? c.invites : [],
createdAt: c.createdAt || new Date(node.ts).toISOString(),
updatedAt: c.updatedAt || null,
tribeId: c.tribeId || null,
encrypted: c.encrypted === true,
undecryptable: !!f._undec,
rid: roomId ? (liveKey ? deriveRid(liveKey, roomId) : "") : f.rid,
token: typeof c.token === "string" ? c.token : "",
hub: FEED_ID.test(String(c.hub || "")) ? c.hub : author,
hubAddress: typeof c.hubAddress === "string" ? c.hubAddress : "",
media: c.media === "video" ? "video" : "audio",
maxParticipants: ROOM_MAX,
number: roomNumberOf(rootId),
line: c.line === "SWITCHBOARD" ? "SWITCHBOARD" : "DND"
}
}
const collectRooms = async (idx) => {
const tribeKeyCache = new Map()
const items = []
for (const [rootId, tipId] of idx.tipByRoot.entries()) {
if (idx.tomb.has(tipId)) continue
const node = idx.nodes.get(tipId)
if (!node || node.c.type !== "room") continue
let tKeys = []
if (node.c.tribeId) {
if (!tribeKeyCache.has(node.c.tribeId)) tribeKeyCache.set(node.c.tribeId, await getTribeKeysFor(node.c.tribeId))
tKeys = tribeKeyCache.get(node.c.tribeId)
}
const room = buildRoom(node, rootId, tKeys, idx.resolveMembers(rootId))
if (!room) continue
room.isClosed = room.status === "CLOSED"
items.push(room)
}
return items
}
const pickHub = async () => {
const ph = await phone()
const me = (await openSsb()).id
const h = ph && typeof ph.roomHubFor === "function" ? await rpcValue(ph.roomHubFor) : null
if (h && FEED_ID.test(String(h.key || ""))) return { hub: h.key, hubAddress: String(h.address || "") }
return { hub: me, hubAddress: "" }
}
const tokenFor = async (rid) => {
const ph = await phone()
const t = ph && typeof ph.roomToken === "function" ? await rpcValue(ph.roomToken, rid) : null
return typeof t === "string" ? t : ""
}
const republish = async (id, mutate) => {
const self = api
const tipId = await self.resolveCurrentId(id)
const ssbClient = await openSsb()
const userId = ssbClient.id
const rootId = await self.resolveRootId(id)
const item = await asP(ssbClient.get, tipId).catch(() => null)
if (!item || !item.content) throw new Error("Room not found")
if (item.content.author !== userId) throw new Error("Not the author")
const c = item.content
let keyHex = null
let usesTribeKey = false
if (c.encrypted === true) {
if (c.tribeId) {
const tKeys = await getTribeKeysFor(c.tribeId)
if (tKeys.length) { keyHex = tKeys[0]; usesTribeKey = true }
}
if (!keyHex) keyHex = lookupKey(rootId)
if (!keyHex) throw new Error("Missing room key — cannot update room")
}
const enc = (text) => (c.encrypted === true ? encryptField(String(text), keyHex) : String(text))
const updated = { ...mutate(c, enc), updatedAt: new Date().toISOString(), replaces: tipId }
await asP(ssbClient.publish, { type: "tombstone", target: tipId, deletedAt: new Date().toISOString(), author: userId })
const res = await asP(ssbClient.publish, updated)
if (keyHex && !usesTribeKey) setRoomKey(res.key, keyHex)
return res
}
const api = {
type: "room",
ROOM_MAX,
async decryptContent(content, rootId) {
const tKeys = content && content.tribeId ? await getTribeKeysFor(content.tribeId) : []
return decryptRoomFields(content, rootId, tKeys)
},
decryptContentPublicSync(content) {
if (!content) return null
if (content.encrypted !== true) return { title: safeText(content.title), description: safeText(content.description), tags: normalizeTags(content.tags) }
if (content.tribeId) return null
const keyHex = tryDecryptPublicInviteKey(content.invites)
if (!keyHex) return null
const f = decryptWithKeys(content, [keyHex])
return f ? { title: f.title, description: f.description, tags: f.tags } : null
},
async resolveRootId(id) {
const ssbClient = await openSsb()
const idx = buildIndex(await readAll(ssbClient))
const root = idx.groupRootOf(id)
if (idx.tomb.has(idx.contentTipOf(root))) throw new Error("Not found")
return root
},
async resolveCurrentId(id) {
const ssbClient = await openSsb()
const idx = buildIndex(await readAll(ssbClient))
const tip = idx.contentTipOf(idx.groupRootOf(id))
if (idx.tomb.has(tip)) throw new Error("Not found")
return tip
},
async createRoom({ title, description, image, status, tags, tribeId } = {}) {
const ssbClient = await openSsb()
const userId = ssbClient.id
const now = new Date().toISOString()
const type = tribeId ? "INVITE-ONLY" : normalizeStatus(status)
const tagsArr = normalizeTags(tags)
const rid = crypto.randomBytes(16).toString("hex")
const roomId = crypto.randomBytes(16).toString("hex")
const sealed = !!tribeId || type !== "OPEN"
const live = { token: await tokenFor(sealed ? roomId : rid), ...(await pickHub()), media: "audio" }
const base = { type: "room", status: tribeId ? "OPEN" : type, author: userId, members: [userId], invites: [], createdAt: now, updatedAt: now, ...live }
if (type === "OPEN" && !tribeId) {
return asP(ssbClient.publish, { ...base, title: safeText(title), description: safeText(description), image: blobRef(image), tags: tagsArr, rid, encrypted: false })
}
let keyHex = null
let usesTribeKey = false
if (tribeId) {
const tKeys = await getTribeKeysFor(tribeId)
if (tKeys.length) { keyHex = tKeys[0]; usesTribeKey = true }
}
if (!keyHex) keyHex = crypto.randomBytes(32).toString("hex")
const enc = (text) => encryptField(String(text), keyHex)
const msg = await asP(ssbClient.publish, {
...base,
title: enc(safeText(title)),
description: enc(safeText(description)),
image: enc(blobRef(image)),
tags: enc(tagsArr.join(",")),
roomId,
encrypted: true,
...(tribeId ? { tribeId } : {})
})
if (!usesTribeKey) {
setRoomKey(msg.key, keyHex)
if (tribeCrypto) {
try {
const ssbKeys = require("../server/node_modules/ssb-keys")
await asP(ssbClient.publish, { type: "tribe-keys", tribeId: msg.key, generation: 1, memberKeys: { [userId]: tribeCrypto.boxKeyForMember(keyHex, userId, ssbKeys) } })
} catch (_) {}
}
}
return msg
},
async updateRoomById(id, data = {}) {
const hub = data.refreshHub ? await pickHub() : null
return republish(id, (c, enc) => ({
...c,
title: data.title !== undefined ? enc(safeText(data.title)) : c.title,
description: data.description !== undefined ? enc(safeText(data.description)) : c.description,
image: data.image !== undefined ? enc(blobRef(data.image)) : c.image,
tags: data.tags !== undefined ? (c.encrypted === true ? enc(normalizeTags(data.tags).join(",")) : normalizeTags(data.tags)) : c.tags,
invites: data.invites !== undefined ? data.invites : c.invites,
...(hub || {})
}))
},
async rehomeHub(id) {
const ssbClient = await openSsb()
const picked = await pickHub()
if (!picked || picked.hub === ssbClient.id) return null
return republish(id, (c) => ({ ...c, ...picked }))
},
async closeRoomById(id) {
return republish(id, (c) => ({ ...c, status: "CLOSED" }))
},
async setRoomLine(id, line) {
return republish(id, (c) => ({ ...c, line: line === "SWITCHBOARD" ? "SWITCHBOARD" : "DND" }))
},
async deleteRoomById(id) {
const tipId = await this.resolveCurrentId(id)
const ssbClient = await openSsb()
const item = await asP(ssbClient.get, tipId).catch(() => null)
if (!item || !item.content) throw new Error("Room not found")
if (item.content.author !== ssbClient.id) throw new Error("Not the author")
await asP(ssbClient.publish, { type: "tombstone", target: tipId, deletedAt: new Date().toISOString(), author: ssbClient.id })
},
async addMemberToRoom(roomId, feedId, consumedCode) {
const tipId = await this.resolveCurrentId(roomId)
const ssbClient = await openSsb()
const rootId = await this.resolveRootId(roomId)
const item = await asP(ssbClient.get, tipId).catch(() => null)
if (!item || !item.content) throw new Error("Room not found")
const c = item.content
if (c.encrypted === true && !c.tribeId && !lookupKey(rootId)) {
const key = tryDecryptPublicInviteKey(c.invites)
if (key) setRoomKey(rootId, key)
}
const proofKey = feedId === ssbClient.id && c.encrypted === true && !c.tribeId ? lookupKey(rootId) : null
return asP(ssbClient.publish, { type: "roomMember", target: rootId, member: feedId, on: true, createdAt: new Date().toISOString(), ...(typeof consumedCode === "string" && consumedCode ? { code: consumedCode } : {}), ...(proofKey ? { keyProof: keyProofFor(proofKey, feedId) } : {}) })
},
async leaveRoomMembership(roomId) {
const ssbClient = await openSsb()
const userId = ssbClient.id
const room = await this.getRoomById(roomId)
if (!room) throw new Error("Room not found")
if (room.author === userId) throw new Error("Author cannot leave their own room")
if (!room.members.includes(userId)) return
await asP(ssbClient.publish, { type: "roomMember", target: room.rootId, member: userId, on: false, createdAt: new Date().toISOString() })
},
async ingestKeys() { await ingestOwnTribeKeys() },
async pruneOrphanKeys() {
if (!ownCrypto || typeof ownCrypto.getAllRootIds !== "function") return 0
try {
const ssbClient = await openSsb()
const tomb = buildValidatedTombstoneSet(await readAll(ssbClient))
let removed = 0
for (const rid of ownCrypto.getAllRootIds()) {
if (tomb.has(rid)) { try { ownCrypto.dropKey(rid); removed += 1 } catch (_) {} }
}
return removed
} catch (_) { return 0 }
},
async getRoomById(id) {
const ssbClient = await openSsb()
const idx = memoIndex('rooms', await readAll(ssbClient), buildIndex)
const root = idx.groupRootOf(id)
const tip = idx.contentTipOf(root)
if (idx.tomb.has(tip)) return null
const node = idx.nodes.get(tip)
if (!node || node.c.type !== "room") return null
const tKeys = node.c.tribeId ? await getTribeKeysFor(node.c.tribeId) : []
const room = buildRoom(node, root, tKeys, idx.resolveMembers(root))
if (!room) return null
room.isClosed = room.status === "CLOSED"
return roomCollab.fold(room, await collectRooms(idx))
},
async listAll({ filter = "all", viewerId } = {}) {
const ssbClient = await openSsb()
const uid = viewerId || ssbClient.id
const messages = await readAll(ssbClient)
const idx = buildIndex(messages)
let list = roomCollab.visibleThenCollapsed(await collectRooms(idx), uid)
try { await ensureMemberKeys(ssbClient, messages, list) } catch (_) {}
if (filter === "mine") list = list.filter(r => r.author === uid)
else if (filter === "recent") list = list.filter(r => new Date(r.createdAt).getTime() >= Date.now() - RECENT_MS)
else if (filter === "open") list = list.filter(r => r.type === "OPEN" && !r.isClosed)
else if (filter === "invite") list = list.filter(r => r.type === "INVITE-ONLY" && !r.isClosed)
else if (filter === "closed") list = list.filter(r => r.isClosed)
return list.slice().sort((a, b) => new Date(b.createdAt) - new Date(a.createdAt))
},
async generateInvite(roomId, opts = {}) {
const ssbClient = await openSsb()
const room = await this.getRoomById(roomId)
if (!room) throw new Error("Room not found")
if (room.author !== ssbClient.id) throw new Error("Only the author can generate invites")
const rootId = await this.resolveRootId(roomId)
const keyHex = lookupKey(rootId)
const code = crypto.randomBytes(INVITE_BYTES).toString("hex")
let invite = code
const pubFlag = opts.public ? { public: true } : {}
const inviteSalt = generateInviteSalt()
if (keyHex) {
invite = opts.public
? { code, ek: encryptForInvite(keyHex, code, inviteSalt), salt: inviteSalt, ...pubFlag }
: { ch: hashCode(code, inviteSalt), ek: encryptForInvite(keyHex, code, inviteSalt), salt: inviteSalt }
} else {
invite = opts.public ? { code, public: true } : { ch: hashCode(code, inviteSalt), salt: inviteSalt }
}
await this.updateRoomById(roomId, { invites: [...room.invites, invite] })
return code
},
async getOpenInvite(roomId) {
return openInviteOf(await this.getRoomById(roomId).catch(() => null))
},
async generateOpenInvite(roomId) {
const room = await this.getRoomById(roomId)
if (!room) throw new Error("Room not found")
if ((Array.isArray(room.invites) ? room.invites : []).find(inv => typeof inv === "object" && inv.public === true)) throw new Error("An open invitation already exists")
return this.generateInvite(roomId, { public: true })
},
async removeOpenInvite(roomId) {
const ssbClient = await openSsb()
const room = await this.getRoomById(roomId)
if (!room) throw new Error("Room not found")
if (room.author !== ssbClient.id) throw new Error("Only the author can remove invites")
await this.updateRoomById(roomId, { invites: (Array.isArray(room.invites) ? room.invites : []).filter(inv => !(typeof inv === "object" && inv.public === true)) })
await rotateRoomKey(await this.resolveRootId(roomId), [...new Set([room.author, ...(Array.isArray(room.members) ? room.members : [])])])
},
async joinByInvite(code) {
const ssbClient = await openSsb()
const userId = ssbClient.id
const rooms = await collectRooms(buildIndex(await readAll(ssbClient)))
let matchedRoom = null
let matchedInvite = null
for (const r of rooms) {
for (const inv of r.invites) {
if (inviteMatches(inv, code)) { matchedRoom = r; matchedInvite = inv; break }
}
if (matchedRoom) break
}
if (!matchedRoom) throw new Error("Invalid or expired invite code")
if (matchedRoom.members.includes(userId)) throw new Error("Already a member")
const isPublic = typeof matchedInvite === "object" && matchedInvite.public === true
const resolvedRootId = await this.resolveRootId(matchedRoom.rootId)
const token = inviteToken(matchedInvite) || code
if (!isPublic) {
const idx = buildIndex(await readAll(ssbClient))
if (idx.isCodeConsumed(resolvedRootId, token) || idx.isCodeConsumed(resolvedRootId, code)) throw new Error("Invite already used")
}
let roomKey = null
if (typeof matchedInvite === "object" && matchedInvite.ek) {
roomKey = decryptFromInvite(matchedInvite.ek, code, matchedInvite.salt)
if (roomKey) setRoomKey(resolvedRootId, roomKey)
}
await this.addMemberToRoom(matchedRoom.rootId, userId, isPublic ? "" : token)
if (tribeCrypto && roomKey) {
try {
const ssbKeys = require("../server/node_modules/ssb-keys")
const memberKeys = {}
try { memberKeys[userId] = tribeCrypto.boxKeyForMember(roomKey, userId, ssbKeys) } catch (_) {}
if (matchedRoom.author && matchedRoom.author !== userId) {
try { memberKeys[matchedRoom.author] = tribeCrypto.boxKeyForMember(roomKey, matchedRoom.author, ssbKeys) } catch (_) {}
}
if (Object.keys(memberKeys).length) await asP(ssbClient.publish, { type: "tribe-keys", tribeId: resolvedRootId, generation: 1, memberKeys })
} catch (_) {}
}
return matchedRoom.rootId
},
async liveParams(room) {
if (!room || !RID.test(String(room.rid || "")) || !room.token) return null
let secrets = null
if (room.tribeId) secrets = (await getTribeKeysFor(room.tribeId)).slice(0, 1)
else if (room.type === "INVITE-ONLY") secrets = lookupKeys(room.rootId).slice(0, 1)
return {
rid: room.rid, owner: room.author, token: room.token, hub: room.hub, address: room.hubAddress,
ref: room.rootId, title: room.title, secrets: Array.isArray(secrets) && secrets.length ? secrets : null,
...(room.roomId ? { roomId: room.roomId } : {})
}
},
async occupancy(room) {
const ph = await phone()
if (!ph || typeof ph.roomCount !== "function" || !room || !RID.test(String(room.rid || ""))) return null
try { return await asP(ph.roomCount, { rid: room.rid, hub: room.hub }) } catch (_) { return null }
},
async occupancies(rooms) {
const out = new Map()
await Promise.all((rooms || []).map(async (r) => { const o = await this.occupancy(r); if (o) out.set(r.rootId, o) }))
return out
},
async join(room) {
const ph = await phone()
if (!ph || typeof ph.roomJoin !== "function") throw new Error("unavailable")
const params = await this.liveParams(room)
if (!params) throw new Error("invalid")
return asP(ph.roomJoin, params)
},
async leave() {
const ph = await phone()
if (ph && typeof ph.roomLeave === "function") await asP(ph.roomLeave).catch(() => null)
},
async mute(flag) {
const ph = await phone()
if (!ph || typeof ph.roomMute !== "function") return null
return asP(ph.roomMute, !!flag)
},
async hand(flag) {
const ph = await phone()
if (!ph || typeof ph.roomHand !== "function") return null
return asP(ph.roomHand, !!flag)
},
async silence(id, flag) {
const ph = await phone()
if (!ph || typeof ph.silence !== "function") return null
return asP(ph.silence, { id, on: !!flag })
},
async camera(flag) {
const ph = await phone()
if (!ph || typeof ph.camera !== "function") return null
return asP(ph.camera, !!flag)
},
async recStart() {
const ph = await phone()
if (!ph || typeof ph.roomRecStart !== "function") return null
return asP(ph.roomRecStart)
},
async recStop() {
const ph = await phone()
if (!ph || typeof ph.roomRecStop !== "function") return null
return asP(ph.roomRecStop)
},
async notify(flag) {
const ph = await phone()
if (!ph || typeof ph.roomNotify !== "function") return null
return asP(ph.roomNotify, !!flag)
},
async clearEvents() {
const ph = await phone()
if (!ph || typeof ph.roomClearEvents !== "function") return null
return asP(ph.roomClearEvents)
},
async liveState() {
const ph = await phone()
return ph && typeof ph.roomState === "function" ? rpcValue(ph.roomState) : null
},
async findByNumber(value, viewerId) {
const number = normalizeNumber(value)
if (!number) return []
return (await this.listAll({ filter: "all", viewerId })).filter(r => r.number === number && !r.isClosed)
}
}
return api
}

1423
src/models/school_model.js Normal file
View file

File diff suppressed because it is too large Load diff

94
src/models/search_model.js
View file

@ -2,6 +2,9 @@ const pull = require('../server/node_modules/pull-stream');
const moment = require('../server/node_modules/moment');
const { getConfig } = require('../configs/config-manager.js');
const { buildValidatedTombstoneSet } = require('./tombstone_validator');
const { readTyped, CONTENT_TYPES } = require('./typed_log');
const longText = require('../backend/long_text');
const { versionGroups, versionEntry } = require('../backend/dedupe');
const logLimit = getConfig().ssbLogStream?.limit || 1000;
module.exports = ({ cooler, padsModel, tribeCrypto, tribesModel }) => {
@ -39,8 +42,8 @@ module.exports = ({ cooler, padsModel, tribeCrypto, tribesModel }) => {
const searchableTypes = [
'post', 'about', 'curriculum', 'tribe', 'transfer', 'feed',
'votes', 'report', 'task', 'event', 'bookmark', 'document',
'image', 'audio', 'video', 'torrent', 'market', 'bankWallet', 'bankClaim',
'project', 'job', 'housing', 'industry', 'industryBlueprint', 'forum', 'vote', 'contact', 'pub', 'map', 'shop', 'shopProduct', 'chat', 'pad', 'poll'
'image', 'audio', 'video', 'torrent', 'file', 'market', 'bankWallet', 'bankClaim',
'project', 'job', 'housing', 'industry', 'industryBlueprint', 'forum', 'vote', 'contact', 'pub', 'map', 'shop', 'shopProduct', 'chat', 'pad', 'room', 'poll', 'schoolCourse', 'wikiPage', 'emergency', 'mailingList', 'logisticsRoute', 'podcast', 'podcastEpisode', 'campaign'
];
const getRelevantFields = (type, content) => {
@ -66,6 +69,7 @@ module.exports = ({ cooler, padsModel, tribeCrypto, tribesModel }) => {
case 'document':
return [content?.url, content?.title, content?.description, ...(content?.tags || []), content?.key];
case 'torrent':
case 'file':
return [content?.title, content?.description, ...(content?.tags || []), content?.url];
case 'market':
return [content?.item_type, content?.title, content?.description, content?.price, ...(content?.tags || []), content?.status, content?.item_status, content?.deadline, content?.includesShipping, content?.seller, content?.image, content?.auctions_poll, content?.stock];
@ -83,6 +87,8 @@ module.exports = ({ cooler, padsModel, tribeCrypto, tribesModel }) => {
return [content?.address];
case 'bankClaim':
return [content?.amount, content?.epochId, content?.allocationId, content?.txid];
case 'schoolCourse':
return [content?.title, content?.description, ...(content?.tags || []), content?.price, content?.status, content?.visibility];
case 'project':
return [content?.title, content?.status, content?.progress, content?.goal, content?.pledged, content?.deadline, (content?.followers || []).length, (content?.backers || []).length, (content?.milestones || []).length, content?.bounty, content?.bountyAmount, content?.bounty_currency, content?.activity?.kind, content?.activityActor];
case 'poll':
@ -115,6 +121,22 @@ module.exports = ({ cooler, padsModel, tribeCrypto, tribesModel }) => {
return [content?.title, content?.description, content?.category, ...(content?.tags || []), content?.status, content?.author];
case 'pad':
return [content?.title, content?.status, content?.deadline, ...(content?.tags || []), content?.author];
case 'room':
return content?.encrypted === true ? [] : [content?.title, content?.description, content?.status, ...(content?.tags || []), content?.author];
case 'wikiPage':
return [content?.title, content?.body, content?.slug, ...(content?.tags || []), ...(content?.aliases || []), content?.author];
case 'emergency':
return [content?.title, content?.text, content?.category, content?.status, ...(content?.tags || []), content?.author];
case 'mailingList':
return [content?.title, content?.description, content?.listType, content?.status, ...(content?.tags || []), content?.author];
case 'logisticsRoute':
return [content?.title, content?.description, content?.origin, content?.destination, content?.kind, content?.mode, ...(content?.tags || []), content?.author];
case 'podcast':
return [content?.title, content?.description, content?.category, ...(content?.tags || []), content?.author];
case 'podcastEpisode':
return [content?.title, content?.description, ...(content?.tags || []), content?.author];
case 'campaign':
return [content?.title, content?.text, content?.category, content?.status, ...(content?.tags || []), content?.author];
case 'gameScore':
return [content?.game, content?.player];
default:
@ -144,6 +166,7 @@ module.exports = ({ cooler, padsModel, tribeCrypto, tribesModel }) => {
if (t === 'audio') return `audio:${c.url || `${author}|${norm(c.title)}|${norm(c.description)}` || msg.key}`;
if (t === 'video') return `video:${c.url || `${author}|${norm(c.title)}|${norm(c.description)}` || msg.key}`;
if (t === 'torrent') return `torrent:${c.url || `${author}|${norm(c.title)}|${norm(c.description)}` || msg.key}`;
if (t === 'file') return `file:${c.url || `${author}|${norm(c.title)}|${norm(c.description)}` || msg.key}`;
if (t === 'bookmark') return `bookmark:${author}|${c.url || norm(c.description) || msg.key}`;
if (t === 'tribe') {
@ -228,6 +251,9 @@ module.exports = ({ cooler, padsModel, tribeCrypto, tribesModel }) => {
].join('|');
}
if (t === 'schoolCourse') {
return ['schoolCourse', author, norm(c.title), norm(c.createdAt)].join('|');
}
if (t === 'project') {
return [
'project',
@ -288,6 +314,30 @@ module.exports = ({ cooler, padsModel, tribeCrypto, tribesModel }) => {
return ['pad', author, norm(c.title), norm(c.deadline)].join('|');
}
if (t === 'room') {
return ['room', author, norm(c.rid || c.title), norm(c.createdAt)].join('|');
}
if (t === 'wikiPage') {
return ['wikiPage', norm(c.tribeId), norm(c.slug)].join('|');
}
if (t === 'emergency') {
return ['emergency', author, norm(c.title), norm(c.createdAt)].join('|');
}
if (t === 'mailingList') {
return ['mailingList', author, norm(c.createdAt)].join('|');
}
if (t === 'logisticsRoute') {
return ['logisticsRoute', author, norm(c.createdAt)].join('|');
}
if (t === 'podcast' || t === 'podcastEpisode' || t === 'campaign') {
return [t, author, norm(c.createdAt)].join('|');
}
return `${t}:${msg.key}`;
};
@ -303,20 +353,14 @@ module.exports = ({ cooler, padsModel, tribeCrypto, tribesModel }) => {
return Array.from(map.values());
};
const search = async ({ query, types = [], resultsPerPage = "10" }) => {
const search = async ({ query, types = [] }) => {
const ssbClient = await openSsb();
const viewerId = ssbClient.id;
const queryLower = String(query || '').toLowerCase();
const messages = await new Promise((res, rej) => {
pull(
ssbClient.createLogStream({ limit: logLimit }),
pull.collect((err, msgs) => err ? rej(err) : res(msgs))
);
});
const messages = await readTyped(ssbClient, CONTENT_TYPES, { limit: logLimit, withWindow: true });
const tombstoned = buildValidatedTombstoneSet(messages);
const replacesMap = new Map();
const latestByKey = new Map();
for (const msg of messages) {
@ -324,23 +368,12 @@ module.exports = ({ cooler, padsModel, tribeCrypto, tribesModel }) => {
const c = msg?.value?.content;
const t = c?.type;
if (!t || !searchableTypes.includes(t)) continue;
if (tombstoned.has(k)) continue;
if (c.replaces) replacesMap.set(c.replaces, k);
latestByKey.set(k, msg);
}
for (const [oldId, newId] of Array.from(replacesMap.entries())) {
const orig = latestByKey.get(oldId);
const repl = latestByKey.get(newId);
if (!orig) { replacesMap.delete(oldId); continue; }
if (!repl || String(repl?.value?.author) !== String(orig?.value?.author)) {
replacesMap.delete(oldId);
latestByKey.delete(newId);
}
}
for (const oldId of replacesMap.keys()) {
latestByKey.delete(oldId);
const vg = versionGroups(Array.from(latestByKey.values()).map(versionEntry));
for (const k of Array.from(latestByKey.keys())) {
if (!vg.roots.has(vg.rootOf(k)) || vg.tipOf(k) !== k || tombstoned.has(k)) latestByKey.delete(k);
}
const viewerTribeIds = new Set();
@ -376,6 +409,14 @@ module.exports = ({ cooler, padsModel, tribeCrypto, tribesModel }) => {
}
}
const chunkLookup = longText.lookupIn(longText.indexChunks(messages));
for (const [k, msg] of Array.from(latestByKey.entries())) {
const c = msg?.value?.content;
if ((c?.type === 'post' || c?.type === 'forum') && longText.hasChunks(c)) {
latestByKey.set(k, { ...msg, value: { ...msg.value, content: longText.resolveField(c, 'text', msg.value.author, chunkLookup) } });
}
}
if (padsModel) {
for (const msg of latestByKey.values()) {
const c = msg?.value?.content;
@ -422,11 +463,6 @@ module.exports = ({ cooler, padsModel, tribeCrypto, tribesModel }) => {
return acc;
}, {});
if (resultsPerPage !== "all") {
const limit = parseInt(resultsPerPage, 10);
for (const key in grouped) grouped[key] = grouped[key].slice(0, limit);
}
return grouped;
};

175
src/models/shops_model.js
View file

@ -3,6 +3,8 @@ const { getConfig } = require("../configs/config-manager.js")
const categories = require("../backend/opinion_categories")
const { buildValidatedTombstoneSet } = require('./tombstone_validator');
const { dedupeByPreferring, norm } = require('../backend/dedupe')
const { readTyped, memoIndex } = require('./typed_log')
const FileType = require('../server/node_modules/file-type')
const logLimit = getConfig().ssbLogStream?.limit || 1000
const safeArr = (v) => (Array.isArray(v) ? v : [])
@ -18,31 +20,90 @@ module.exports = ({ cooler, tribeCrypto }) => {
let ssb
const openSsb = async () => { if (!ssb) ssb = await cooler.open(); return ssb }
const readAll = async (ssbClient) =>
new Promise((resolve, reject) =>
pull(ssbClient.createLogStream({ limit: logLimit }), pull.collect((err, msgs) => err ? reject(err) : resolve(msgs)))
)
const SHOP_TYPES = ["shop", "shopProduct", "shopOpinion", "shopPurchase", "shop-invite", "shop-invite-tombstone", "shop-open-invite", "shop-open-invite-tombstone", "tombstone"]
const readAll = async (ssbClient) => readTyped(ssbClient, SHOP_TYPES, { limit: logLimit, withWindow: true, withPrivate: true })
const mediaKinds = new Map()
const blobMediaKind = async (ssbClient, blobId) => {
if (mediaKinds.has(blobId)) return mediaKinds.get(blobId)
let kind
try {
const head = await new Promise((resolve, reject) => pull(ssbClient.blobs.get(blobId), pull.take(1), pull.collect((err, chunks) => err ? reject(err) : resolve(Buffer.concat(chunks)))))
const ft = await FileType.fromBuffer(head)
kind = ft && ft.mime ? ft.mime.split('/')[0] : 'other'
} catch (_) { return null }
mediaKinds.set(blobId, kind)
return kind
}
const withMediaKind = async (ssbClient, items) => {
for (const p of items) {
if (!p || !p.image || !String(p.image).startsWith('&')) continue
const kind = await blobMediaKind(ssbClient, p.image)
if (kind === 'video') p.image = `[video:](${p.image})`
else if (kind === 'audio') p.image = `[audio:](${p.image})`
}
return items
}
const openSealed = (ssbClient, m) => {
if (typeof m.value?.content === "string") return ssbClient.private.unbox({ key: m.key, value: m.value, timestamp: m.value?.timestamp || m.timestamp || 0 })
return m.value?.private === true ? m : null
}
const SELLER_STATUSES = ["ACCEPTED", "REJECTED", "PAID", "SHIPPED"]
const BUYER_STATUSES = ["RECEIVED"]
const ORDER_STATUSES = [...SELLER_STATUSES, ...BUYER_STATUSES]
const ORDER_FLOW = { PENDING: ["ACCEPTED", "REJECTED"], ACCEPTED: ["PAID", "REJECTED"], PAID: ["SHIPPED"], SHIPPED: ["RECEIVED"] }
const buildOrderStatusMap = (ssbClient, messages) => {
const map = new Map()
for (const m of messages) {
if (typeof m.value?.content !== "string") continue
try {
const dec = ssbClient.private.unbox({ key: m.key, value: m.value, timestamp: m.value?.timestamp || m.timestamp || 0 })
const dec = openSealed(ssbClient, m)
const dc = dec?.value?.content
if (!dc || dc.type !== "shop-purchase-status" || !dc.orderId) continue
const ts = dec.value.timestamp || m.timestamp || 0
const prev = map.get(dc.orderId)
if (!prev || ts >= prev.ts) map.set(dc.orderId, { status: String(dc.status || "").toUpperCase(), ts })
if (!map.has(dc.orderId)) map.set(dc.orderId, [])
map.get(dc.orderId).push({ status: String(dc.status || "").toUpperCase(), author: dec.value.author, ts: dec.value.timestamp || m.timestamp || 0, price: dc.price !== undefined ? String(dc.price) : undefined, title: dc.title !== undefined ? String(dc.title) : undefined })
} catch (_) {}
}
return map
}
const orderStatusOf = (updates, buyer, seller) => {
const signed = (updates || []).filter(u => (SELLER_STATUSES.includes(u.status) && u.author === seller) || (BUYER_STATUSES.includes(u.status) && u.author === buyer)).sort((a, b) => a.ts - b.ts)
let status = "PENDING", g = 0
while (g++ < 100) {
const next = signed.find(u => (ORDER_FLOW[status] || []).includes(u.status))
if (!next) break
status = next.status
}
return status
}
const acceptedTermsOf = (updates, seller) => {
const accepted = (updates || []).filter(u => u.status === "ACCEPTED" && u.author === seller && u.price !== undefined).sort((a, b) => a.ts - b.ts)[0]
return accepted ? { price: accepted.price, ...(accepted.title ? { title: accepted.title } : {}) } : null
}
const withOrderStatus = (statusMap) => (o) => {
const updates = statusMap.get(o.id)
return { ...o, ...(acceptedTermsOf(updates, o.seller) || {}), status: orderStatusOf(updates, o.buyer, o.seller) }
}
const resolveOrder = (idx, dc) => {
const productRoot = idx.rootOf(String(dc.productId || ""))
const productNode = idx.nodes.get(productRoot)
if (!productNode || productNode.c.type !== "shopProduct") return null
const shopRoot = idx.rootOf(productShopId(productNode.c))
const shopNode = idx.nodes.get(shopRoot)
if (!shopNode || shopNode.c.type !== "shop" || shopNode.author !== productNode.author) return null
let ref = productRoot
let g = 0
while (idx.strictChild.has(ref) && g++ < 100000) ref = idx.strictChild.get(ref)
const raw = idx.nodes.get(ref).c
const c = isEncrypted(raw) ? decryptShopContent(raw, shopRoot) : raw
const readable = !c._undecryptable
return { productId: productRoot, shopId: shopRoot, seller: shopNode.author, title: readable ? String(c.title || "") : "", price: readable ? (c.price || "") : "" }
}
const decryptBuyers = (val, key) => {
if (Array.isArray(val)) return val
if (typeof val === 'string' && tribeCrypto && key) {
@ -77,7 +138,7 @@ module.exports = ({ cooler, tribeCrypto }) => {
}
const buildIndex = (messages) => {
const tomb = new Set()
const tomb = buildValidatedTombstoneSet(messages)
const nodes = new Map()
const parent = new Map()
const child = new Map()
@ -90,7 +151,7 @@ module.exports = ({ cooler, tribeCrypto }) => {
const v = m.value || {}
const c = v.content
if (!c) continue
if (c.type === "tombstone" && c.target) { tomb.add(c.target); continue }
if (c.type === "tombstone") continue
if (c.type === "shopOpinion" && c.target) { opinionMsgs.push({ target: c.target, author: v.author, category: c.category }); continue }
if (c.type === "shopPurchase" && c.target) { purchaseMsgs.push({ target: c.target, author: v.author }); continue }
if (c.type === "shop" || c.type === "shopProduct") {
@ -162,7 +223,7 @@ module.exports = ({ cooler, tribeCrypto }) => {
tags: safeArr(c.tags),
visibility: c.visibility || (encrypted ? "CLOSED" : "OPEN"),
clearnetPublic: !!c.clearnetPublic,
author: raw.author || c.author || node.author,
author: node.author,
createdAt: c.createdAt || raw.createdAt || new Date(node.ts).toISOString(),
updatedAt: c.updatedAt || raw.updatedAt || null,
opinions: agg.opinions,
@ -182,7 +243,6 @@ module.exports = ({ cooler, tribeCrypto }) => {
const undecryptable = !!c._undecryptable
const agg = idx ? idx.aggregateFor(rootId, c) : { opinions: c.opinions || {}, opinions_inhabitants: safeArr(c.opinions_inhabitants) }
const sellerStock = Number(c.stock) || 0
const purchaseCount = idx ? idx.purchaseCountFor(rootId) : 0
const sellerBuyers = encrypted ? safeArr(c.buyers) : decryptBuyers(c.buyers, tribeCrypto ? tribeCrypto.getKey(rootId) : null)
const buyers = Array.from(new Set(sellerBuyers.concat(idx ? idx.purchaseAuthorsFor(rootId) : [])))
return {
@ -193,9 +253,9 @@ module.exports = ({ cooler, tribeCrypto }) => {
description: c.description || "",
image: c.image || null,
price: c.price || "0.000000",
stock: Math.max(0, sellerStock - purchaseCount),
stock: Math.max(0, sellerStock),
featured: !!c.featured,
author: raw.author || c.author || node.author,
author: node.author,
createdAt: c.createdAt || raw.createdAt || new Date(node.ts).toISOString(),
updatedAt: c.updatedAt || raw.updatedAt || null,
opinions: agg.opinions,
@ -346,7 +406,7 @@ module.exports = ({ cooler, tribeCrypto }) => {
async getShopById(id) {
const ssbClient = await openSsb()
const messages = await readAll(ssbClient)
const idx = buildIndex(messages)
const idx = memoIndex('shops', messages, buildIndex)
const tip = idx.strictTipOf(id)
if (idx.tomb.has(tip)) return null
@ -521,7 +581,7 @@ module.exports = ({ cooler, tribeCrypto }) => {
async getProductById(id) {
const ssbClient = await openSsb()
const messages = await readAll(ssbClient)
const idx = buildIndex(messages)
const idx = memoIndex('shops', messages, buildIndex)
const tip = idx.strictTipOf(id)
if (idx.tomb.has(tip)) return null
@ -531,7 +591,7 @@ module.exports = ({ cooler, tribeCrypto }) => {
const root = idx.rootOf(tip)
return buildProduct(node, root, idx)
return (await withMediaKind(ssbClient, [buildProduct(node, root, idx)]))[0]
},
async listProducts(shopRootId) {
@ -549,7 +609,7 @@ module.exports = ({ cooler, tribeCrypto }) => {
if (prod && !prod.undecryptable) items.push(prod)
}
return items.sort((a, b) => new Date(b.createdAt) - new Date(a.createdAt))
return withMediaKind(ssbClient, items.sort((a, b) => new Date(b.createdAt) - new Date(a.createdAt)))
},
async listFeaturedProducts(shopRootId) {
@ -568,7 +628,7 @@ module.exports = ({ cooler, tribeCrypto }) => {
if (prod && !prod.undecryptable) items.push(prod)
}
return items.sort((a, b) => new Date(b.createdAt) - new Date(a.createdAt)).slice(0, 4)
return withMediaKind(ssbClient, items.sort((a, b) => new Date(b.createdAt) - new Date(a.createdAt)).slice(0, 4))
},
async listAllProducts({ filter = "all", sort = "recent" } = {}) {
@ -585,8 +645,10 @@ module.exports = ({ cooler, tribeCrypto }) => {
if (prod && !prod.undecryptable) items.push(prod)
}
if (filter === "top") return items.slice().sort((a, b) => voteSum(b.opinions) - voteSum(a.opinions) || new Date(b.createdAt) - new Date(a.createdAt))
return items.slice().sort((a, b) => new Date(b.createdAt) - new Date(a.createdAt))
const sorted = filter === "top"
? items.slice().sort((a, b) => voteSum(b.opinions) - voteSum(a.opinions) || new Date(b.createdAt) - new Date(a.createdAt))
: items.slice().sort((a, b) => new Date(b.createdAt) - new Date(a.createdAt))
return withMediaKind(ssbClient, sorted)
},
async buyProduct(productId) {
@ -608,8 +670,7 @@ module.exports = ({ cooler, tribeCrypto }) => {
const shopRoot = productShopId(raw)
const c = encrypted ? decryptShopContent(raw, shopRoot) : raw
if (c._undecryptable) throw new Error("Cannot access product")
const stock = Math.max(0, (Number(c.stock) || 0) - idx.purchaseCountFor(rootId))
if (stock <= 0) throw new Error("Out of stock")
if (!((Number(c.stock) || 0) > 0)) throw new Error("Out of stock")
const content = { type: "shopPurchase", target: rootId, createdAt: new Date().toISOString() }
return new Promise((res, rej) => ssbClient.publish(content, (e, m) => e ? rej(e) : res(m)))
@ -626,11 +687,14 @@ module.exports = ({ cooler, tribeCrypto }) => {
const rootId = idx.rootOf(tipId)
const node = idx.nodes.get(tipId)
if (!node) throw new Error("Product not found")
if (!node || node.c.type !== "shopProduct") throw new Error("Product not found")
const raw = node.c
const shopOwner = raw.author
if (shopOwner === userId) throw new Error("Cannot buy your own product")
const shopRoot = productShopId(raw)
const shopNode = idx.nodes.get(idx.rootOf(shopRoot))
const productRootNode = idx.nodes.get(rootId)
if (!shopNode || shopNode.c.type !== "shop" || !productRootNode || productRootNode.author !== shopNode.author || node.author !== shopNode.author) throw new Error("Product not found")
const shopOwner = shopNode.author
if (shopOwner === userId) throw new Error("Cannot buy your own product")
const c = isEncrypted(raw) ? decryptShopContent(raw, shopRoot) : raw
const content = {
@ -655,31 +719,24 @@ module.exports = ({ cooler, tribeCrypto }) => {
const ssbClient = await openSsb()
const me = ssbClient.id
const messages = await readAll(ssbClient)
const idx = buildIndex(messages)
const out = []
for (const m of messages) {
if (typeof m.value?.content !== "string") continue
try {
const dec = ssbClient.private.unbox({ key: m.key, value: m.value, timestamp: m.value?.timestamp || m.timestamp || 0 })
const dec = openSealed(ssbClient, m)
if (!dec?.value?.content) continue
const dc = dec.value.content
if (dc.type !== "shop-purchase") continue
if (dec.value.author !== me) continue
out.push({ id: m.key, ...dc, buyer: dec.value.author, ts: dec.value.timestamp || m.timestamp || 0 })
const ts = dec.value.timestamp || m.timestamp || 0
const resolved = resolveOrder(idx, dc)
if (!resolved) continue
out.push({ id: m.key, ...dc, ...resolved, buyer: dec.value.author, ts })
} catch (_) {}
}
const statusMap = buildOrderStatusMap(ssbClient, messages)
const sellerCache = new Map()
for (const o of out) {
if (!o.seller && o.shopId) {
if (!sellerCache.has(o.shopId)) {
const shop = await this.getShopById(o.shopId).catch(() => null)
sellerCache.set(o.shopId, (shop && shop.author) || null)
}
o.seller = sellerCache.get(o.shopId) || null
}
}
return out
.map(o => ({ ...o, status: (statusMap.get(o.id) || {}).status || "PENDING" }))
.map(withOrderStatus(statusMap))
.sort((a, b) => b.ts - a.ts)
},
@ -689,23 +746,26 @@ module.exports = ({ cooler, tribeCrypto }) => {
const shop = await this.getShopById(shopRootId).catch(() => null)
if (!shop) throw new Error("Shop not found")
if (shop.author !== me) throw new Error("Not the shop owner")
const rootId = shop.rootId || shopRootId
const messages = await readAll(ssbClient)
const idx = buildIndex(messages)
const out = []
for (const m of messages) {
if (typeof m.value?.content !== "string") continue
try {
const dec = ssbClient.private.unbox({ key: m.key, value: m.value, timestamp: m.value?.timestamp || m.timestamp || 0 })
const dec = openSealed(ssbClient, m)
if (!dec?.value?.content) continue
const dc = dec.value.content
if (dc.type !== "shop-purchase") continue
if (dc.shopId !== shopRootId) continue
out.push({ id: m.key, ...dc, buyer: dec.value.author, ts: dec.value.timestamp || m.timestamp || 0 })
const ts = dec.value.timestamp || m.timestamp || 0
const resolved = resolveOrder(idx, dc)
if (!resolved || resolved.shopId !== rootId || resolved.seller !== me) continue
out.push({ id: m.key, ...dc, ...resolved, buyer: dec.value.author, ts })
} catch (_) {}
}
const statusMap = buildOrderStatusMap(ssbClient, messages)
return out
.map(o => ({ ...o, status: (statusMap.get(o.id) || {}).status || "PENDING" }))
.map(withOrderStatus(statusMap))
.sort((a, b) => b.ts - a.ts)
},
@ -715,27 +775,38 @@ module.exports = ({ cooler, tribeCrypto }) => {
const st = String(status || "").toUpperCase()
if (!ORDER_STATUSES.includes(st)) throw new Error("Invalid status")
const messages = await readAll(ssbClient)
const idx = buildIndex(messages)
let order = null
for (const m of messages) {
if (m.key !== orderId) continue
if (typeof m.value?.content !== "string") break
try {
const dec = ssbClient.private.unbox({ key: m.key, value: m.value, timestamp: m.value?.timestamp || m.timestamp || 0 })
const dec = openSealed(ssbClient, m)
const dc = dec?.value?.content
if (dc && dc.type === "shop-purchase") order = { ...dc, buyer: dec.value.author }
const resolved = dc && dc.type === "shop-purchase" ? resolveOrder(idx, dc) : null
if (resolved) order = { ...dc, ...resolved, buyer: dec.value.author }
} catch (_) {}
break
}
if (!order) throw new Error("Order not found")
const seller = order.seller || (await this.getShopById(order.shopId).catch(() => null) || {}).author || null
const seller = order.seller
if (BUYER_STATUSES.includes(st)) {
if (me !== order.buyer) throw new Error("Only the buyer can set this status")
} else {
if (me !== seller) throw new Error("Only the seller can set this status")
}
const current = orderStatusOf(buildOrderStatusMap(ssbClient, messages).get(orderId), order.buyer, seller)
if (!(ORDER_FLOW[current] || []).includes(st)) throw new Error("Invalid status transition")
const recps = Array.from(new Set([order.buyer, seller].filter(Boolean)))
const content = { type: "shop-purchase-status", orderId, shopId: order.shopId, status: st, updatedAt: new Date().toISOString() }
return new Promise((res, rej) => ssbClient.private.publish(content, recps, (e, m) => e ? rej(e) : res(m)))
const content = { type: "shop-purchase-status", orderId, shopId: order.shopId, status: st, ...(st === "ACCEPTED" ? { price: String(order.price || ""), title: String(order.title || "") } : {}), updatedAt: new Date().toISOString() }
const published = await new Promise((res, rej) => ssbClient.private.publish(content, recps, (e, m) => e ? rej(e) : res(m)))
const delta = st === "ACCEPTED" ? -1 : (st === "REJECTED" && current === "ACCEPTED" ? 1 : 0)
if (delta) {
try {
const product = await this.getProductById(order.productId)
await this.updateProductById(order.productId, { stock: String(Math.max(0, (Number(product.stock) || 0) + delta)) })
} catch (_) {}
}
return published
},
async countPendingOrders(shopRootId) {

145
src/models/stats_model.js
View file

@ -1,16 +1,19 @@
const pull = require('../server/node_modules/pull-stream');
const os = require('os');
const fs = require('fs');
const path = require('path');
const ssbDirOf = () => require('../server/ssb_config').path;
const { getConfig } = require('../configs/config-manager.js');
const { buildValidatedTombstoneSet } = require('./tombstone_validator');
const { readTyped, CONTENT_TYPES } = require('./typed_log');
const { versionGroups, versionEntry } = require('../backend/dedupe');
const logLimit = getConfig().ssbLogStream?.limit || 1000;
const DISK_USAGE_TTL_MS = 10 * 60 * 1000;
const STORAGE_DIR = path.join(__dirname, "..", "configs");
const ADDR_FILE = path.join(STORAGE_DIR, "wallet_addresses.json");
const addrFile = () => process.env.OASIS_BANKING_DIR ? path.join(process.env.OASIS_BANKING_DIR, "wallet-addresses.json") : require('../configs/state-manager').statePath("wallet-addresses.json");
function readAddrMap() {
try {
const ADDR_FILE = addrFile();
if (!fs.existsSync(ADDR_FILE)) return {};
const raw = fs.readFileSync(ADDR_FILE, 'utf8');
const obj = JSON.parse(raw || '{}');
@ -22,7 +25,7 @@ function readAddrMap() {
const listPubsFromEbt = () => {
try {
const ebtDir = path.join(require('../server/ssb_config').path, 'ebt');
const ebtDir = path.join(ssbDirOf(), 'ebt');
const files = fs.readdirSync(ebtDir);
return files.filter(f => f.endsWith('.ed25519'));
} catch {
@ -59,25 +62,51 @@ module.exports = ({ cooler, tribeCrypto, tribesModel }) => {
const types = [
'bookmark','event','task','votes','report','feed','project','industry','industryBlueprint',
'image','torrent','audio','video','document','transfer','post','tribe',
'image','torrent','file','audio','video','document','transfer','post','tribe',
'market','forum','job','aiExchange','map','shop','shopProduct','chat','chatMessage',
'pad','padEntry','gameScore','calendar','calendarDate','calendarNote','log',
'pad','padEntry','room','roomMember','wikiPage','emergency','emergencyConfirm','emergencyUpdate','mailingList','logisticsRoute','logisticsRating','podcast','podcastEpisode','podcastPlay','campaign','campaignSignature','campaignUpdate','gameScore','calendar','calendarDate','calendarNote','log',
'schoolCourse','schoolLesson','schoolEnroll','schoolCertificate',
'parliamentCandidature','parliamentTerm','parliamentProposal','parliamentRevocation','parliamentLaw',
'courtsCase','courtsEvidence','courtsAnswer','courtsVerdict','courtsSettlement','courtsSettlementProposal','courtsSettlementAccepted','courtsNomination','courtsNominationVote'
];
const getFolderSize = (folderPath) => {
let files;
try { files = fs.readdirSync(folderPath); } catch (_) { return 0; }
let totalSize = 0;
for (const file of files) {
const filePath = `${folderPath}/${file}`;
const walkBytes = async (dir) => {
let names;
try { names = await fs.promises.readdir(dir); } catch (_) { return 0; }
let total = 0;
for (const name of names) {
const filePath = path.join(dir, name);
try {
const st = fs.statSync(filePath);
totalSize += st.isDirectory() ? getFolderSize(filePath) : st.size;
const st = await fs.promises.stat(filePath);
total += st.isDirectory() ? await walkBytes(filePath) : st.size;
} catch (_) {}
}
return totalSize;
return total;
};
const diskUsage = new Map();
const ssbDiskUsage = (root) => {
const hit = diskUsage.get(root);
if (hit && (hit.pending || Date.now() - hit.at < DISK_USAGE_TTL_MS)) return hit.pending || Promise.resolve(hit.value);
const pending = (async () => {
let names = [];
try { names = await fs.promises.readdir(root); } catch (_) {}
const usage = { total: 0, log: 0, blobs: 0 };
for (const name of names) {
const filePath = path.join(root, name);
let bytes = 0;
try {
const st = await fs.promises.stat(filePath);
bytes = st.isDirectory() ? await walkBytes(filePath) : st.size;
} catch (_) {}
usage.total += bytes;
if (name === 'db2') usage.log = bytes;
if (name === 'blobs') usage.blobs = bytes;
}
return usage;
})();
diskUsage.set(root, { pending, at: 0, value: null });
pending.then((value) => diskUsage.set(root, { pending: null, at: Date.now(), value }), () => diskUsage.delete(root));
return pending;
};
const formatSize = (sizeInBytes) => {
@ -274,9 +303,31 @@ module.exports = ({ cooler, tribeCrypto, tribesModel }) => {
else if (t === 'larphousepost') score += 6;
else if (t === 'larptestattempt') score += 3;
else if (t === 'torrent') score += 6;
else if (t === 'file') score += 6;
else if (t === 'shop' || t === 'shopproduct') score += 6;
else if (t === 'schoolcourse' || rawType === 'schoolcourse') score += 10;
else if (t === 'schoollesson' || rawType === 'schoollesson') score += 6;
else if (t === 'schoolenroll' || rawType === 'schoolenroll') score += 3;
else if (t === 'schoolcertificate' || rawType === 'schoolcertificate') score += 12;
else if (t === 'schoolexam' || rawType === 'schoolexam') score += 6;
else if (t === 'schoolprogress' || rawType === 'schoolprogress') score += 1;
else if (t === 'schoolopinion' || rawType === 'schoolopinion') score += 2;
else if (t === 'shop-purchase') score += 2;
else if (t === 'pad' || t === 'padentry') score += 3;
else if (t === 'room') score += 3;
else if (t === 'roommember') score += 1;
else if (t === 'wikipage') score += 3;
else if (t === 'emergency') score += 4;
else if (t === 'emergencyconfirm' || t === 'emergencyupdate') score += 1;
else if (t === 'mailinglist') score += 3;
else if (t === 'logisticsroute') score += 3;
else if (t === 'logisticsrating') score += 1;
else if (t === 'podcast') score += 4;
else if (t === 'podcastepisode') score += 3;
else if (t === 'podcastopinion' || t === 'podcastplay') score += 1;
else if (t === 'campaign') score += 4;
else if (t === 'campaignsignature') score += 1;
else if (t === 'campaignupdate') score += 2;
else if (t === 'calendar' || t === 'calendarnote' || t === 'calendardate') score += 3;
else if (t === 'chat') score += 1;
else if (t === 'gamescore') score += 2;
@ -289,6 +340,19 @@ module.exports = ({ cooler, tribeCrypto, tribesModel }) => {
else if (rawType === 'industryvote') score += 2;
else if (rawType === 'industrymember') score += 2;
else if (rawType === 'industryopinion') score += 1;
else if (rawType === 'wikipage') score += c.replaces ? 2 : 6;
else if (rawType === 'emergency') score += c.replaces ? 1 : 5;
else if (rawType === 'emergencyconfirm') score += 1;
else if (rawType === 'emergencyupdate') score += 2;
else if (rawType === 'mailinglist') score += c.replaces ? 1 : 3;
else if (rawType === 'logisticsroute') score += c.replaces ? 1 : 4;
else if (rawType === 'logisticsrating' || rawType === 'logisticsopinion') score += 1;
else if (rawType === 'podcast') score += c.replaces ? 1 : 5;
else if (rawType === 'podcastepisode') score += c.replaces ? 1 : 4;
else if (rawType === 'podcastopinion' || rawType === 'podcastplay') score += 1;
else if (rawType === 'campaign') score += c.replaces ? 1 : 5;
else if (rawType === 'campaignsignature' || rawType === 'campaignopinion') score += 1;
else if (rawType === 'campaignupdate') score += 2;
}
return Math.max(0, Math.round(score));
};
@ -297,12 +361,7 @@ module.exports = ({ cooler, tribeCrypto, tribesModel }) => {
const ssbClient = await openSsb();
const userId = ssbClient.id;
const messages = await new Promise((res, rej) => {
pull(
ssbClient.createLogStream({ limit: logLimit, reverse: true }),
pull.collect((err, msgs) => err ? rej(err) : res(msgs))
);
});
const messages = (await readTyped(ssbClient, CONTENT_TYPES, { limit: logLimit, withWindow: true })).reverse();
const allMsgs = messages.filter(m => {
const c = m.value && m.value.content;
@ -316,15 +375,15 @@ module.exports = ({ cooler, tribeCrypto, tribesModel }) => {
const scopedMsgs = filter === 'MINE' ? allMsgs.filter(m => m.value.author === userId) : allMsgs;
const byType = {};
const parentOf = {};
const versionsOf = {};
for (const t of types) {
byType[t] = new Map();
parentOf[t] = new Map();
versionsOf[t] = [];
}
const fpIdx = tribeCrypto ? tribeCrypto.buildFingerprintIndex() : null;
const accessibleTribeIds = await buildAccessibleTribeIds();
for (const m of scopedMsgs) {
for (const m of allMsgs) {
const k = m.key;
let c = m.value.content;
if (tribeCrypto && tribeCrypto.isTribeMsg(c)) {
@ -344,27 +403,20 @@ module.exports = ({ cooler, tribeCrypto, tribesModel }) => {
let theType = c.type;
if (!types.includes(theType)) continue;
byType[theType].set(k, { key: k, ts: m.value.timestamp, content: c, author: m.value.author });
if (c.replaces) parentOf[theType].set(k, c.replaces);
versionsOf[theType].push(versionEntry({ key: k, value: { ...m.value, content: c } }));
}
const findRoot = (t, id) => {
let cur = id;
const pMap = parentOf[t];
while (pMap.has(cur)) cur = pMap.get(cur);
return cur;
};
const vgOf = {};
for (const t of types) vgOf[t] = versionGroups(versionsOf[t]);
const findRoot = (t, id) => vgOf[t].rootOf(id);
const tipOf = {};
for (const t of types) {
tipOf[t] = new Map();
const pMap = parentOf[t];
const fwd = new Map();
for (const [child, parent] of pMap.entries()) fwd.set(parent, child);
const allMap = byType[t];
const roots = new Set(Array.from(allMap.keys()).map(id => findRoot(t, id)));
for (const root of roots) {
let tip = root;
while (fwd.has(tip)) tip = fwd.get(tip);
for (const root of vgOf[t].roots) {
if (filter === 'MINE' && allMap.get(root).author !== userId) continue;
const tip = vgOf[t].tipOf(root);
if (tombTargets.has(tip)) continue;
const node = allMap.get(tip) || allMap.get(root);
if (node) tipOf[t].set(root, node);
@ -395,6 +447,11 @@ module.exports = ({ cooler, tribeCrypto, tribesModel }) => {
.filter(n => n.content?.isAnonymous !== false && Array.isArray(n.content?.members) && n.content.members.includes(userId))
.map(n => ({ id: findRoot('tribe', n.key), name: n.content.name || n.content.title || n.key }));
const opinionTargets = new Set();
for (const m of allMsgs) {
const c = m && m.value && m.value.content;
if (c && typeof c.type === 'string' && c.type.endsWith('Opinion') && typeof c.target === 'string') opinionTargets.add(c.target);
}
const content = {};
const opinions = {};
for (const t of types) {
@ -406,7 +463,9 @@ module.exports = ({ cooler, tribeCrypto, tribesModel }) => {
if (t === 'forum') vals = vals.filter(c => !(c.root && tombTargets.has(c.root)));
}
content[t] = vals.length || 0;
opinions[t] = vals.filter(e => Array.isArray(e.opinions_inhabitants) && e.opinions_inhabitants.length > 0).length || 0;
const embedded = vals.filter(e => Array.isArray(e.opinions_inhabitants) && e.opinions_inhabitants.length > 0).length || 0;
const external = t === 'tribe' ? 0 : Array.from(tipOf[t].keys()).filter(root => vgOf[t].groups.get(root).some(id => opinionTargets.has(id))).length;
opinions[t] = Math.max(embedded, external);
}
if (filter === 'MINE') {
@ -428,12 +487,10 @@ module.exports = ({ cooler, tribeCrypto, tribesModel }) => {
const inhabitants = new Set(allMsgs.map(m => m.value.author)).size;
const secretStat = fs.statSync((require('../server/ssb_config').path + '/secret'));
const secretStat = fs.statSync(path.join(ssbDirOf(), 'secret'));
const createdAt = secretStat.birthtime.toLocaleString();
const folderSize = getFolderSize((require('../server/ssb_config').path + ''));
const flumeSize = getFolderSize((require('../server/ssb_config').path + '/flume'));
const blobsSize = getFolderSize((require('../server/ssb_config').path + '/blobs'));
const { total: folderSize, log: logSize, blobs: blobsSize } = await ssbDiskUsage(ssbDirOf());
const allTs = scopedMsgs.map(m => m.value.timestamp || 0).filter(Boolean);
const lastTs = allTs.length ? Math.max(...allTs) : 0;
@ -572,7 +629,7 @@ module.exports = ({ cooler, tribeCrypto, tribesModel }) => {
userTombstoneCount: scopedMsgs.filter(m => m.value.content.type === 'tombstone' && m.value.author === userId).length,
networkTombstoneCount: validatedTombstoneCount,
folderSize: formatSize(folderSize),
statsBlockchainSize: formatSize(flumeSize),
statsBlockchainSize: formatSize(logSize),
statsBlobsSize: formatSize(blobsSize),
pubsCount,
activity: {

104
src/models/subscriptions_model.js Normal file
View file

@ -0,0 +1,104 @@
const { getConfig } = require('../configs/config-manager.js');
const { readTyped } = require('./typed_log');
const logLimit = getConfig().ssbLogStream?.limit || 1000;
module.exports = ({ cooler }) => {
let ssb;
const openSsb = async () => { if (!ssb) ssb = await cooler.open(); return ssb; };
const latestByTargetAuthor = (messages) => {
const latest = new Map();
for (const m of messages) {
const c = m?.value?.content;
if (!c || c.type !== 'subscription' || typeof c.target !== 'string') continue;
const author = m.value.author;
const ts = m.value.timestamp || 0;
const k = `${c.target}::${author}`;
const prev = latest.get(k);
if (!prev || ts >= prev.ts) latest.set(k, { ts, target: c.target, scope: String(c.scope || ''), author, on: c.on !== false });
}
return latest;
};
return {
async setSubscription(target, scope, on) {
const ssbClient = await openSsb();
if (typeof target !== 'string' || !target.trim()) throw new Error('Invalid target');
const content = {
type: 'subscription',
target: target.trim(),
scope: String(scope || '').trim().slice(0, 32),
on: on !== false,
updatedAt: new Date().toISOString()
};
return new Promise((res, rej) => ssbClient.publish(content, (e, m) => (e ? rej(e) : res(m))));
},
async myState(target) {
const ssbClient = await openSsb();
const messages = await readTyped(ssbClient, ['subscription'], { limit: logLimit });
const latest = latestByTargetAuthor(messages);
const mine = latest.get(`${target}::${ssbClient.id}`);
if (!mine) return null;
return mine.on ? 'on' : 'off';
},
async isSubscribed(target) {
const ssbClient = await openSsb();
const messages = await readTyped(ssbClient, ['subscription'], { limit: logLimit });
const latest = latestByTargetAuthor(messages);
const mine = latest.get(`${target}::${ssbClient.id}`);
return !!(mine && mine.on);
},
async listSubscribers(target) {
const ssbClient = await openSsb();
const messages = await readTyped(ssbClient, ['subscription'], { limit: logLimit });
const latest = latestByTargetAuthor(messages);
const out = [];
for (const entry of latest.values()) {
if (entry.target === target && entry.on) out.push(entry.author);
}
return Array.from(new Set(out));
},
async mySubscriptions() {
const ssbClient = await openSsb();
const messages = await readTyped(ssbClient, ['subscription'], { limit: logLimit });
const latest = latestByTargetAuthor(messages);
const out = [];
for (const entry of latest.values()) {
if (entry.author === ssbClient.id && entry.on) out.push({ target: entry.target, scope: entry.scope });
}
return out;
},
async subscribersMap(targets) {
const ssbClient = await openSsb();
const messages = await readTyped(ssbClient, ['subscription'], { limit: logLimit });
const latest = latestByTargetAuthor(messages);
const out = new Map();
for (const t of targets || []) out.set(t, []);
for (const entry of latest.values()) {
if (!out.has(entry.target) || !entry.on) continue;
out.get(entry.target).push(entry.author);
}
return out;
},
async subscriberCounts(targets) {
const ssbClient = await openSsb();
const messages = await readTyped(ssbClient, ['subscription'], { limit: logLimit });
const latest = latestByTargetAuthor(messages);
const counts = new Map();
for (const t of targets || []) counts.set(t, 0);
const mine = new Set();
for (const entry of latest.values()) {
if (!counts.has(entry.target) || !entry.on) continue;
counts.set(entry.target, counts.get(entry.target) + 1);
if (entry.author === ssbClient.id) mine.add(entry.target);
}
return { counts, mine };
}
};
};

27
src/models/tags_model.js
View file

@ -1,6 +1,8 @@
const pull = require('../server/node_modules/pull-stream');
const { getConfig } = require('../configs/config-manager.js');
const { buildValidatedTombstoneSet } = require('./tombstone_validator');
const { readTyped, CONTENT_TYPES } = require('./typed_log');
const { versionGroups, versionEntry } = require('../backend/dedupe');
const logLimit = getConfig().ssbLogStream?.limit || 1000;
module.exports = ({ cooler, padsModel, tribesModel }) => {
@ -108,16 +110,10 @@ module.exports = ({ cooler, padsModel, tribesModel }) => {
async listTags(filter = 'all', search = '') {
const ssbClient = await openSsb();
const messages = await new Promise((resolve, reject) => {
pull(
ssbClient.createLogStream({ limit: logLimit }),
pull.collect((err, msgs) => err ? reject(err) : resolve(msgs))
);
});
const messages = await readTyped(ssbClient, CONTENT_TYPES, { limit: logLimit, withWindow: true });
const tombstoned = buildValidatedTombstoneSet(messages);
const replacesMap = new Map();
const latestByKey = new Map();
for (const msg of messages) {
@ -125,24 +121,15 @@ module.exports = ({ cooler, padsModel, tribesModel }) => {
const c = msg?.value?.content;
const t = c?.type;
if (!k || !c || !t) continue;
if (tombstoned.has(k)) continue;
if (t === 'tombstone') continue;
if (c.replaces) replacesMap.set(c.replaces, k);
latestByKey.set(k, msg);
}
for (const [oldId, newId] of Array.from(replacesMap.entries())) {
const oldMsg = latestByKey.get(oldId);
if (!oldMsg) { replacesMap.delete(oldId); continue; }
const newMsg = latestByKey.get(newId);
if (!newMsg || String(newMsg?.value?.author) !== String(oldMsg?.value?.author)) {
replacesMap.delete(oldId);
latestByKey.delete(newId);
}
const vg = versionGroups(Array.from(latestByKey.values()).map(versionEntry));
for (const k of Array.from(latestByKey.keys())) {
if (!vg.roots.has(vg.rootOf(k)) || vg.tipOf(k) !== k || tombstoned.has(k)) latestByKey.delete(k);
}
for (const oldId of replacesMap.keys()) latestByKey.delete(oldId);
const viewerId = ssbClient.id;
const viewerVisibleTribeIds = new Set();
if (tribesModel) {
@ -216,7 +203,7 @@ module.exports = ({ cooler, padsModel, tribesModel }) => {
if (filter === 'mine') {
tags = tags.filter(t => (t.mine || 0) > 0).sort((a, b) => b.mine - a.mine || a.name.localeCompare(b.name));
} else if (filter === 'recent') {
const since = Date.now() - 7 * DAY;
const since = Date.now() - DAY;
tags = tags.filter(t => (t.lastTs || 0) >= since).sort((a, b) => (b.lastTs || 0) - (a.lastTs || 0));
} else if (filter === 'top') {
tags.sort((a, b) => b.count - a.count || a.name.localeCompare(b.name));

69
src/models/tasks_model.js
View file

@ -3,7 +3,8 @@ const moment = require('../server/node_modules/moment');
const { normalizeImages, normalizeVideo } = require('./media_gallery');
const { getConfig } = require('../configs/config-manager.js');
const { buildValidatedTombstoneSet } = require('./tombstone_validator');
const { dedupeBy, norm } = require('../backend/dedupe');
const { readTyped, memoIndex } = require('./typed_log');
const { dedupeBy, norm, versionGroups } = require('../backend/dedupe');
const logLimit = getConfig().ssbLogStream?.limit || 1000;
module.exports = ({ cooler, pmModel }) => {
@ -23,13 +24,9 @@ module.exports = ({ cooler, pmModel }) => {
return fallback;
};
const getAllMessages = async (ssbClient) =>
new Promise((resolve, reject) => {
pull(
ssbClient.createLogStream({ limit: logLimit }),
pull.collect((err, msgs) => (err ? reject(err) : resolve(msgs)))
);
});
const TASK_TYPES = ['task', 'taskOpinion', 'taskAssign', 'taskReminderSent', 'tombstone'];
const getAllMessages = async (ssbClient) => readTyped(ssbClient, TASK_TYPES, { limit: logLimit });
const getMsg = async (ssbClient, key) =>
new Promise((resolve, reject) => {
@ -39,9 +36,6 @@ module.exports = ({ cooler, pmModel }) => {
const buildIndex = (messages) => {
const tomb = buildValidatedTombstoneSet(messages);
const nodes = new Map();
const parent = new Map();
const child = new Map();
const strictChild = new Map();
const opinionMsgs = [];
const assignMsgs = [];
@ -57,34 +51,12 @@ module.exports = ({ cooler, pmModel }) => {
if (c.type !== 'task') continue;
const ts = v.timestamp || m.timestamp || 0;
nodes.set(k, { key: k, ts, c, author: v.author });
if (c.replaces) {
parent.set(k, c.replaces);
child.set(c.replaces, k);
}
nodes.set(k, { key: k, ts, seq: v.sequence || 0, c, author: v.author });
}
for (const [k, node] of nodes) {
const t = node.c.replaces;
if (t) { const orig = nodes.get(t); if (orig && orig.author === node.author) strictChild.set(t, k); }
}
const rootOf = (id) => {
let cur = id, g = 0;
while (parent.has(cur) && nodes.has(parent.get(cur)) && g++ < 100000) cur = parent.get(cur);
return cur;
};
const contentTipOf = (root) => {
let cur = root, g = 0;
while (strictChild.has(cur) && g++ < 100000) cur = strictChild.get(cur);
const n = nodes.get(cur), rn = nodes.get(root);
return (n && rn && n.author === rn.author) ? cur : root;
};
const roots = new Set();
for (const id of nodes.keys()) roots.add(rootOf(id));
const vg = versionGroups([...nodes.values()].map(n => ({ key: n.key, author: n.author, seq: n.seq, ts: n.ts, replaces: n.c.replaces || null, claimedAuthor: typeof n.c.author === 'string' ? n.c.author : null })));
const rootOf = vg.rootOf;
const contentTipOf = vg.tipOf;
const opinionsByRoot = new Map();
for (const op of opinionMsgs) { if (!nodes.has(op.target)) continue; const r = rootOf(op.target); if (!opinionsByRoot.has(r)) opinionsByRoot.set(r, []); opinionsByRoot.get(r).push(op); }
@ -117,9 +89,9 @@ module.exports = ({ cooler, pmModel }) => {
};
const tipByRoot = new Map();
for (const r of roots) tipByRoot.set(r, contentTipOf(r));
for (const r of vg.roots) tipByRoot.set(r, contentTipOf(r));
return { tomb, nodes, parent, child, rootOf, tipByRoot, resolveGroup };
return { tomb, nodes, rootOf, tipByRoot, resolveGroup };
};
const buildTask = (node, rootId, agg, now) => {
@ -141,6 +113,11 @@ module.exports = ({ cooler, pmModel }) => {
};
};
const currentTipId = async (ssb, id) => {
const idx = buildIndex(await getAllMessages(ssb));
return idx.tipByRoot.get(idx.rootOf(id)) || id;
};
return {
async createTask(title, description, startTime, endTime, priority, location = '', tagsRaw = [], isPublic, media = {}) {
const ssb = await openSsb();
@ -185,9 +162,10 @@ module.exports = ({ cooler, pmModel }) => {
async deleteTaskById(taskId) {
const ssb = await openSsb();
const userId = ssb.id;
const task = await new Promise((res, rej) => ssb.get(taskId, (err, task) => err ? rej(new Error('Task not found')) : res(task)));
const tipId = await currentTipId(ssb, taskId);
const task = await new Promise((res, rej) => ssb.get(tipId, (err, task) => err ? rej(new Error('Task not found')) : res(task)));
if (task.content.author !== userId) throw new Error('Not the author');
const tombstone = { type: 'tombstone', target: taskId, deletedAt: new Date().toISOString(), author: userId };
const tombstone = { type: 'tombstone', target: tipId, deletedAt: new Date().toISOString(), author: userId };
return new Promise((res, rej) => ssb.publish(tombstone, (err, result) => err ? rej(err) : res(result)));
},
@ -198,8 +176,9 @@ module.exports = ({ cooler, pmModel }) => {
const keys = Object.keys(updatedData || {}).filter(k => updatedData[k] !== undefined);
if (keys.length === 1 && keys[0] === 'assignees') return this.toggleAssignee(taskId);
const tipId = await currentTipId(ssb, taskId);
const old = await new Promise((res, rej) =>
ssb.get(taskId, (err, msg) => err || !msg ? rej(new Error('Task not found')) : res(msg))
ssb.get(tipId, (err, msg) => err || !msg ? rej(new Error('Task not found')) : res(msg))
);
const c = old.content;
@ -262,7 +241,7 @@ module.exports = ({ cooler, pmModel }) => {
video: updatedData.video !== undefined ? normalizeVideo(updatedData.video) : normalizeVideo(c.video),
assignees: nextAssignees,
updatedAt: new Date().toISOString(),
replaces: taskId
replaces: tipId
};
return new Promise((res, rej) => ssb.publish(updated, (err, result) => err ? rej(err) : res(result)));
@ -278,10 +257,10 @@ module.exports = ({ cooler, pmModel }) => {
const ssb = await openSsb();
const now = moment();
const messages = await getAllMessages(ssb);
const idx = buildIndex(messages);
const idx = memoIndex('tasks', messages, buildIndex);
const rootId = idx.rootOf(taskId);
const agg = idx.resolveGroup(rootId);
if (!agg.contentNode || idx.tomb.has(agg.contentTip)) throw new Error('Task not found');
if (!idx.tipByRoot.has(rootId) || !agg.contentNode || idx.tomb.has(agg.contentTip)) throw new Error('Task not found');
return buildTask(agg.contentNode, rootId, agg, now);
},

11
src/models/tombstone_validator.js
View file

@ -2,26 +2,27 @@ function buildValidatedTombstoneSet(messages) {
if (!Array.isArray(messages)) return new Set();
const tombClaims = new Map();
const targetAuthors = new Map();
const targetContentAuthors = new Map();
const targetRoots = new Map();
const claims = [];
for (const m of messages) {
if (!m || !m.value) continue;
const v = m.value;
const c = v.content;
if (!c) continue;
if (typeof c === 'object' && c.type === 'tombstone' && typeof c.target === 'string') {
const ts = v.timestamp || 0;
const prev = tombClaims.get(c.target);
if (!prev || ts > prev.ts) tombClaims.set(c.target, { author: v.author, ts, rootId: c._rootId || null });
claims.push({ target: c.target, author: v.author, rootId: c._rootId || null });
continue;
}
if (m.key) {
targetAuthors.set(m.key, v.author);
if (typeof c === 'object' && c.type === 'feed' && typeof c.author === 'string') targetContentAuthors.set(m.key, c.author);
if (typeof c === 'object' && c._rootId) targetRoots.set(m.key, c._rootId);
}
}
const out = new Set();
for (const [target, { author, rootId }] of tombClaims.entries()) {
if (targetAuthors.get(target) !== author) continue;
for (const { target, author, rootId } of claims) {
if (targetAuthors.get(target) !== author && targetContentAuthors.get(target) !== author) continue;
if (rootId) {
const targetRoot = targetRoots.get(target);
if (targetRoot !== rootId) continue;

169
src/models/torrent_downloads_model.js Normal file
View file

@ -0,0 +1,169 @@
const pull = require('../server/node_modules/pull-stream');
const Abortable = require('../server/node_modules/pull-abortable');
const JOB_TTL_MS = 6 * 60 * 60 * 1000;
const PROBE_TIMEOUT_MS = 8000;
const RECHECK_MS = 15000;
module.exports = ({ cooler }) => {
const jobs = new Map();
let recheckTimer = null;
const openSsb = () => cooler.open();
const withTimeout = (p, ms) => Promise.race([p, new Promise((_, reject) => setTimeout(() => reject(new Error('timeout')), ms))]);
const peerRpcs = (ssb) => {
const out = [];
const live = ssb && ssb.peers && typeof ssb.peers === 'object' ? ssb.peers : {};
for (const id of Object.keys(live)) {
if (id === ssb.id) continue;
for (const rpc of (Array.isArray(live[id]) ? live[id] : [])) {
if (rpc && rpc.blobs && typeof rpc.blobs.get === 'function' && !rpc.closed) out.push({ id, rpc });
}
}
return out;
};
const hasLocal = (ssb, blobId) => new Promise((resolve) => {
try { ssb.blobs.has(blobId, (err, has) => resolve(!err && !!has)); } catch (_) { resolve(false); }
});
const remoteHas = (peer, blobId) => withTimeout(new Promise((resolve) => {
try { peer.rpc.blobs.has(blobId, (err, has) => resolve(!err && !!has)); } catch (_) { resolve(false); }
}), PROBE_TIMEOUT_MS).catch(() => false);
const findSeeds = async (ssb, blobId) => {
const peers = peerRpcs(ssb);
const flags = await Promise.all(peers.map((p) => remoteHas(p, blobId)));
return peers.filter((_, i) => flags[i]);
};
const fetchFrom = (ssb, seed, job) => new Promise((resolve, reject) => {
const abortable = Abortable();
job.abort = () => abortable.abort(new Error('cancelled'));
let received = 0;
const cap = job.size > 0 ? job.size : 75 * 1024 * 1024;
const opts = { key: job.blobId, max: cap };
pull(
seed.rpc.blobs.get(opts),
abortable,
pull.through((buf) => { received += buf.length; job.received = received; if (received > cap) abortable.abort(new Error('too large')); }),
ssb.blobs.add(job.blobId, (err) => { job.abort = null; err ? reject(err) : resolve(received); })
);
});
const finish = (job, state, error) => {
job.state = state;
job.error = error ? String(error.message || error) : '';
job.finishedAt = Date.now();
job.abort = null;
if (state === 'done' && job.size > 0) job.received = job.size;
};
const ensureRecheck = () => {
if (recheckTimer) return;
recheckTimer = setInterval(async () => {
const pending = Array.from(jobs.values()).filter((j) => j.state === 'nosource');
if (!pending.length) { clearInterval(recheckTimer); recheckTimer = null; return; }
let ssb;
try { ssb = await openSsb(); } catch (_) { return; }
for (const job of pending) {
if (await hasLocal(ssb, job.blobId)) { finish(job, 'done'); continue; }
const seeds = await findSeeds(ssb, job.blobId);
if (seeds.length) run(job);
}
}, RECHECK_MS);
if (recheckTimer.unref) recheckTimer.unref();
};
const run = async (job) => {
if (job.state === 'downloading' || job.state === 'probing') return;
let ssb;
try { ssb = await openSsb(); } catch (err) { finish(job, 'failed', err); return; }
if (await hasLocal(ssb, job.blobId)) { finish(job, 'done'); return; }
job.state = 'probing';
job.error = '';
const seeds = await findSeeds(ssb, job.blobId);
job.seeds = seeds.map((s) => s.id);
if (!seeds.length) {
job.state = 'nosource';
try { ssb.blobs.want(job.blobId, () => {}); } catch (_) {}
ensureRecheck();
return;
}
job.state = 'downloading';
job.startedAt = job.startedAt || Date.now();
let lastErr = null;
for (const seed of seeds) {
if (job.cancelled) { finish(job, 'cancelled'); return; }
job.peer = seed.id;
job.received = 0;
try {
await fetchFrom(ssb, seed, job);
finish(job, 'done');
return;
} catch (err) {
lastErr = err;
if (job.cancelled) { finish(job, 'cancelled'); return; }
}
}
finish(job, 'failed', lastErr);
};
const sweep = () => {
const now = Date.now();
for (const [key, job] of jobs) if (job.finishedAt && now - job.finishedAt > JOB_TTL_MS) jobs.delete(key);
};
const snapshot = (job) => ({
blobId: job.blobId,
name: job.name,
size: job.size,
received: Math.min(job.received || 0, job.size || job.received || 0),
state: job.state,
seeds: Array.isArray(job.seeds) ? job.seeds.length : 0,
peer: job.peer || '',
error: job.error || '',
torrentKey: job.torrentKey || '',
startedAt: job.startedAt || 0,
finishedAt: job.finishedAt || 0,
active: job.state === 'queued' || job.state === 'probing' || job.state === 'downloading' || job.state === 'nosource'
});
return {
start({ blobId, size, name, torrentKey } = {}) {
const id = String(blobId || '');
if (!id.startsWith('&')) throw new Error('torrentFromContentMissing');
sweep();
let job = jobs.get(id);
if (job && (job.state === 'downloading' || job.state === 'probing' || job.state === 'done')) return snapshot(job);
job = { blobId: id, size: Number(size) || 0, name: String(name || ''), torrentKey: torrentKey || '', received: 0, state: 'queued', seeds: [], error: '', cancelled: false, startedAt: 0, finishedAt: 0, abort: null, peer: '' };
jobs.set(id, job);
run(job);
return snapshot(job);
},
cancel(blobId) {
const job = jobs.get(String(blobId || ''));
if (!job) return false;
job.cancelled = true;
if (typeof job.abort === 'function') { try { job.abort(); } catch (_) {} }
else if (job.state !== 'done') finish(job, 'cancelled');
return true;
},
remove(blobId) { return jobs.delete(String(blobId || '')); },
get(blobId) {
const job = jobs.get(String(blobId || ''));
return job ? snapshot(job) : null;
},
list() {
sweep();
return Array.from(jobs.values()).map(snapshot).sort((a, b) => (b.startedAt || 0) - (a.startedAt || 0));
},
async hasLocal(blobId) {
try { return await hasLocal(await openSsb(), String(blobId || '')); } catch (_) { return false; }
},
async seedsFor(blobId) {
try { return (await findSeeds(await openSsb(), String(blobId || ''))).map((s) => s.id); } catch (_) { return []; }
}
};
};

87
src/models/torrents_model.js
View file

@ -1,8 +1,9 @@
const pull = require("../server/node_modules/pull-stream");
const { getConfig } = require("../configs/config-manager.js");
const categories = require("../backend/opinion_categories");
const { readTyped, memoIndex } = require("./typed_log");
const { dedupeBy, norm } = require('../backend/dedupe');
const { dedupeBy, norm, versionGroups } = require('../backend/dedupe');
const logLimit = getConfig().ssbLogStream?.limit || 1000;
const safeArr = (v) => (Array.isArray(v) ? v : []);
@ -13,6 +14,10 @@ const normalizeTags = (raw) => {
return String(raw).split(",").map((t) => t.trim()).filter(Boolean);
};
const cipherField = (cipher) => (cipher && typeof cipher === "object" && cipher.key && cipher.manifestBlobId)
? { cipher: { v: Number(cipher.v) || 1, key: String(cipher.key), manifestBlobId: String(cipher.manifestBlobId), chunkCount: Number(cipher.chunkCount) || 0 } }
: {};
const parseBlobId = (blobMarkdown) => {
const s = String(blobMarkdown || "");
const match = s.match(/\((&[^)]+\.sha256)\)/);
@ -40,10 +45,9 @@ module.exports = ({ cooler, tribeCrypto, tribesModel }) => {
const unwrapForIndex = (msgs) => tribeHelpers ? tribeHelpers.unwrapMessagesForKind(msgs, 'torrent') : msgs;
const tombFor = async (target, tribeId, author) => tribeHelpers ? tribeHelpers.encryptTombstone(target, tribeId, author) : { type: 'tombstone', target, deletedAt: new Date().toISOString(), author };
const getAllMessages = async (ssbClient) =>
new Promise((resolve, reject) => {
pull(ssbClient.createLogStream({ limit: logLimit }), pull.collect((err, msgs) => (err ? reject(err) : resolve(msgs))));
});
const TORRENT_TYPES = ["torrent", "torrentOpinion", "tombstone"];
const getAllMessages = async (ssbClient) => readTyped(ssbClient, TORRENT_TYPES, { limit: logLimit, withWindow: true });
const getMsg = async (ssbClient, key) =>
new Promise((resolve) => {
@ -53,9 +57,6 @@ module.exports = ({ cooler, tribeCrypto, tribesModel }) => {
const buildIndex = (messages) => {
const tomb = new Set();
const nodes = new Map();
const parent = new Map();
const child = new Map();
const strictChild = new Map();
const authorByKey = new Map();
const tombRequests = [];
const opinionMsgs = [];
@ -78,49 +79,24 @@ module.exports = ({ cooler, tribeCrypto, tribesModel }) => {
const ts = v.timestamp || m.timestamp || 0;
let sizeBytes = 0;
try { sizeBytes = Buffer.byteLength(JSON.stringify(v), "utf8"); } catch (_) { sizeBytes = 0; }
nodes.set(k, { key: k, ts, c, sizeBytes, author: v.author });
nodes.set(k, { key: k, ts, seq: v.sequence || 0, c, sizeBytes, author: v.author });
authorByKey.set(k, v.author);
if (c.replaces) {
parent.set(k, c.replaces);
child.set(c.replaces, k);
}
}
for (const [k, node] of nodes) {
const t = node.c.replaces;
if (t) { const orig = nodes.get(t); if (orig && orig.author === node.author) strictChild.set(t, k); }
}
const rootOf = (id) => {
let cur = id, g = 0;
while (parent.has(cur) && nodes.has(parent.get(cur)) && g++ < 100000) cur = parent.get(cur);
return cur;
};
const contentTipOf = (root) => {
let cur = root, g = 0;
while (strictChild.has(cur) && g++ < 100000) cur = strictChild.get(cur);
const n = nodes.get(cur), rn = nodes.get(root);
return (n && rn && n.author === rn.author) ? cur : root;
};
const vg = versionGroups([...nodes.values()].map(n => ({ key: n.key, author: n.author, seq: n.seq, ts: n.ts, replaces: n.c.replaces || null, claimedAuthor: typeof n.c.author === "string" ? n.c.author : null })));
const rootOf = vg.rootOf;
const contentTipOf = vg.tipOf;
for (const t of tombRequests) {
const targetAuthor = authorByKey.get(t.target);
if (targetAuthor && t.author === targetAuthor) tomb.add(t.target);
}
const roots = new Set();
for (const id of nodes.keys()) roots.add(rootOf(id));
const opinionsByRoot = new Map();
for (const op of opinionMsgs) { if (!nodes.has(op.target)) continue; const r = rootOf(op.target); if (!opinionsByRoot.has(r)) opinionsByRoot.set(r, []); opinionsByRoot.get(r).push(op); }
const tipByRoot = new Map();
for (const r of roots) tipByRoot.set(r, contentTipOf(r));
const forward = new Map();
for (const [newId, oldId] of parent.entries()) forward.set(oldId, newId);
for (const r of vg.roots) tipByRoot.set(r, contentTipOf(r));
const resolveGroup = (root) => {
const contentTip = contentTipOf(root);
@ -137,7 +113,7 @@ module.exports = ({ cooler, tribeCrypto, tribesModel }) => {
return { contentTip, contentNode, opinions, voters };
};
return { tomb, nodes, parent, child, rootOf, tipByRoot, forward, resolveGroup };
return { tomb, nodes, rootOf, tipByRoot, resolveGroup };
};
const buildTorrent = (node, rootId, viewerId, agg) => {
@ -156,6 +132,10 @@ module.exports = ({ cooler, tribeCrypto, tribesModel }) => {
title: undec ? "" : (c.title || ""),
description: undec ? "" : (c.description || ""),
size: c.size || 0,
source: undec ? "" : (typeof c.source === "string" && c.source.startsWith("&") ? c.source : ""),
sourceName: undec ? "" : (c.sourceName || ""),
cipher: undec ? null : (c.cipher && typeof c.cipher === "object" && typeof c.cipher.key === "string" && typeof c.cipher.manifestBlobId === "string" ? { v: Number(c.cipher.v) || 1, key: c.cipher.key, manifestBlobId: c.cipher.manifestBlobId, chunkCount: Number(c.cipher.chunkCount) || 0 } : null),
sourceMime: undec ? "" : (c.sourceMime || ""),
opinions,
opinions_inhabitants: voters,
hasVoted: viewerId ? voters.includes(viewerId) : false,
@ -184,16 +164,12 @@ module.exports = ({ cooler, tribeCrypto, tribesModel }) => {
const messages = await getAllMessages(ssbClient);
const idx = buildIndex(unwrapForIndex(messages));
let tip = id;
while (idx.forward.has(tip)) tip = idx.forward.get(tip);
if (idx.tomb.has(tip)) throw new Error("Torrent not found");
let root = tip;
while (idx.parent.has(root)) root = idx.parent.get(root);
const root = idx.rootOf(id);
if (idx.tomb.has(idx.tipByRoot.get(root) || id)) throw new Error("Torrent not found");
return root;
},
async createTorrent(blobMarkdown, tagsRaw, title, description, size, tribeId) {
async createTorrent(blobMarkdown, tagsRaw, title, description, size, tribeId, source = null) {
const ssbClient = await openSsb();
const blobId = parseBlobId(blobMarkdown);
const tags = normalizeTags(tagsRaw) || [];
@ -211,7 +187,11 @@ module.exports = ({ cooler, tribeCrypto, tribesModel }) => {
size: Number(size) || 0,
opinions: {},
opinions_inhabitants: [],
...(tribeId ? { tribeId } : {})
...(tribeId ? { tribeId } : {}),
...(source && typeof source.source === "string" && source.source.startsWith("&")
? { source: source.source, sourceName: String(source.sourceName || ""), sourceMime: String(source.sourceMime || "") }
: {}),
...cipherField(source && source.cipher)
};
content = await encryptIfTribe(content);
@ -221,7 +201,7 @@ module.exports = ({ cooler, tribeCrypto, tribesModel }) => {
});
},
async updateTorrentById(id, blobMarkdown, tagsRaw, title, description) {
async updateTorrentById(id, blobMarkdown, tagsRaw, title, description, extra = null) {
const ssbClient = await openSsb();
const userId = ssbClient.id;
const tipId = await this.resolveCurrentId(id);
@ -231,7 +211,7 @@ module.exports = ({ cooler, tribeCrypto, tribesModel }) => {
const oldDec = await decryptIfTribe(oldMsg.content);
if (!oldDec || oldDec.type !== "torrent") throw new Error("Torrent not found");
assertReadable(oldDec, "Torrent");
if ((oldDec.author || oldMsg.content.author) !== userId) throw new Error("Not the author");
if (oldMsg.author !== userId) throw new Error("Not the author");
const aggTorrent = await this.getTorrentById(id, userId);
if (aggTorrent && Object.keys(aggTorrent.opinions || {}).some(k => (aggTorrent.opinions[k] || 0) > 0)) throw new Error("Cannot edit torrent after it has received opinions.");
@ -243,6 +223,7 @@ module.exports = ({ cooler, tribeCrypto, tribesModel }) => {
type: "torrent",
replaces: tipId,
url: blobId || oldDec.url,
...(blobId ? cipherField(extra && extra.cipher) : cipherField(oldDec.cipher)),
tags,
title: title !== undefined ? title || "" : oldDec.title || "",
description: description !== undefined ? description || "" : oldDec.description || "",
@ -250,7 +231,7 @@ module.exports = ({ cooler, tribeCrypto, tribesModel }) => {
opinions: oldDec.opinions || {},
opinions_inhabitants: oldDec.opinions_inhabitants || [],
author: oldDec.author || userId,
...(oldMsg.content.tribeId ? { tribeId: oldMsg.content.tribeId } : {}),
...((oldDec.tribeId || oldMsg.content.tribeId) ? { tribeId: oldDec.tribeId || oldMsg.content.tribeId } : {}),
createdAt: oldDec.createdAt,
updatedAt: now
};
@ -274,7 +255,7 @@ module.exports = ({ cooler, tribeCrypto, tribesModel }) => {
if (!msg) throw new Error("Torrent not found");
const dec = await decryptIfTribe(msg.content);
if (!dec || dec.type !== "torrent") throw new Error("Torrent not found");
if ((dec.author || msg.content.author) !== userId) throw new Error("Not the author");
if (msg.author !== userId) throw new Error("Not the author");
const tombstone = await tombFor(tipId, dec.tribeId || msg.content.tribeId, userId);
@ -338,10 +319,10 @@ module.exports = ({ cooler, tribeCrypto, tribesModel }) => {
const ssbClient = await openSsb();
const viewer = viewerId || ssbClient.id;
const messages = await getAllMessages(ssbClient);
const idx = buildIndex(unwrapForIndex(messages));
await decryptIndexNodes(idx);
const idx = await memoIndex('torrents', messages, async (m) => { const built = buildIndex(unwrapForIndex(m)); await decryptIndexNodes(built); return built; });
const root = idx.rootOf(id);
if (idx.nodes.has(root) && !idx.tipByRoot.has(root)) throw new Error("Torrent not found");
const agg = idx.resolveGroup(root);
if (agg.contentNode && !idx.tomb.has(agg.contentTip)) return buildTorrent(agg.contentNode, root, viewer, agg);

108
src/models/transfers_model.js
View file

@ -1,11 +1,14 @@
const pull = require("../server/node_modules/pull-stream")
const moment = require("../server/node_modules/moment")
const { getConfig } = require("../configs/config-manager.js")
const { getConfig, defaultInvite } = require("../configs/config-manager.js")
const categories = require("../backend/opinion_categories")
const { buildValidatedTombstoneSet } = require('./tombstone_validator');
const { dedupeByPreferring, norm } = require('../backend/dedupe')
const { readTyped, memoIndex } = require('./typed_log')
const logLimit = getConfig().ssbLogStream?.limit || 1000
const TRANSFER_TYPES = ["transfer", "transferConfirm", "transferOpinion", "ubiClaim", "tombstone"]
const isValidId = (to) => /^@[A-Za-z0-9+/]+={0,2}\.ed25519$/.test(String(to || ""))
const parseNum = (v) => {
@ -19,6 +22,11 @@ const normalizeTags = (raw) => {
return String(raw).split(",").map(t => t.trim()).filter(Boolean)
}
const defaultPubId = () => {
const m = String((defaultInvite() || {}).code || "").match(/(@[A-Za-z0-9+/]+={0,2}\.ed25519)/)
return m ? m[1] : ""
}
const CATEGORIES = ["ECONOMIC", "TIME", "TRUST"]
const normalizeCategory = (raw) => {
const c = String(raw || "ECONOMIC").trim().toUpperCase()
@ -28,14 +36,16 @@ const normalizeCategory = (raw) => {
module.exports = ({ cooler }) => {
let ssb
const openSsb = async () => { if (!ssb) ssb = await cooler.open(); return ssb }
const ubiEpochOf = (c) => {
const tags = Array.isArray(c.tags) ? c.tags.map(t => String(t)) : []
const fromTag = tags.find(t => /^epoch:\d{4}-\d{2}$/i.test(t))
if (fromTag) return fromTag.slice(6)
const m = String(c.concept || "").match(/(\d{4}-\d{2})/)
return m ? m[1] : ""
}
const getAllMessages = async (ssbClient) =>
new Promise((resolve, reject) => {
pull(
ssbClient.createLogStream({ limit: logLimit }),
pull.collect((err, msgs) => err ? reject(err) : resolve(msgs))
)
})
readTyped(ssbClient, TRANSFER_TYPES, { limit: logLimit })
const getMsg = async (ssbClient, key) =>
new Promise((resolve, reject) => {
@ -43,7 +53,7 @@ module.exports = ({ cooler }) => {
})
const buildIndex = (messages) => {
const tomb = new Set()
const tomb = buildValidatedTombstoneSet(messages)
const nodes = new Map()
const parent = new Map()
const child = new Map()
@ -51,7 +61,7 @@ module.exports = ({ cooler }) => {
const confirms = []
const opinionMsgs = []
const ubiByPub = new Map()
const ubiByUser = new Map()
const claimedPubs = new Map()
const ubiClaimNodes = []
for (const m of messages) {
@ -60,43 +70,43 @@ module.exports = ({ cooler }) => {
const c = v.content
if (!c) continue
if (c.type === "tombstone" && c.target) {
tomb.add(c.target)
continue
}
if (c.type === "transferConfirm") { confirms.push({ target: c.target, author: v.author, ts: v.timestamp || 0 }); continue }
if (c.type === "tombstone") continue
if (c.type === "transferConfirm") { confirms.push({ target: c.target, tip: typeof c.tip === "string" ? c.tip : null, author: v.author, ts: v.timestamp || 0 }); continue }
if (c.type === "transferOpinion") { opinionMsgs.push({ target: c.target, author: v.author, category: c.category, ts: v.timestamp || 0 }); continue }
if (c.type === "transfer") {
if (c.from !== v.author) continue
nodes.set(k, { key: k, ts: v.timestamp || m.timestamp || 0, c, author: v.author })
if (c.replaces) {
parent.set(k, c.replaces)
child.set(c.replaces, k)
}
const tags = Array.isArray(c.tags) ? c.tags.map(t => String(t).toUpperCase()) : []
if (tags.includes("UBI") && c.to && c.concept) {
const key = `${c.to}::${c.concept}`
if (v.author === c.from) ubiByPub.set(key, k)
else ubiByUser.set(key, k)
if (tags.includes("UBI") && !tags.includes("REBALANCE") && c.to && c.concept) {
const key = `${c.to}::${ubiEpochOf(c) || c.concept}`
if (!ubiByPub.has(key)) ubiByPub.set(key, new Set())
ubiByPub.get(key).add(v.author)
}
}
if (c.type === "ubiClaim") {
ubiClaimNodes.push({ k, v, c, ts: v.timestamp || m.timestamp || 0 })
if (c.pubId) {
if (!claimedPubs.has(v.author)) claimedPubs.set(v.author, new Set())
claimedPubs.get(v.author).add(c.pubId)
}
}
}
for (const [key, userMsgKey] of ubiByUser.entries()) {
if (ubiByPub.has(key)) tomb.add(userMsgKey)
}
const defaultPub = defaultPubId()
for (const { k, v, c, ts } of ubiClaimNodes) {
if (tomb.has(k)) continue
const claimantId = v.author
const epochId = c.epochId || ""
const concept = `UBI ${epochId} ${claimantId}`.trim()
const key = `${claimantId}::${concept}`
if (ubiByPub.has(key) || ubiByUser.has(key)) continue
const concept = `UBI - ${epochId}`.trim()
const key = `${claimantId}::${epochId || concept}`
const payers = ubiByPub.get(key)
if (payers && ((c.pubId && payers.has(c.pubId)) || (defaultPub && payers.has(defaultPub)))) continue
const synthetic = {
type: "transfer",
from: c.pubId || "",
@ -106,7 +116,7 @@ module.exports = ({ cooler }) => {
createdAt: c.claimedAt || new Date(ts).toISOString(),
updatedAt: c.claimedAt || new Date(ts).toISOString(),
deadline: null,
confirmedBy: [c.pubId || ""].filter(Boolean),
confirmedBy: [],
status: "UNCONFIRMED",
tags: ["UBI", "PENDING"],
opinions: {},
@ -153,42 +163,43 @@ module.exports = ({ cooler }) => {
const resolveGroup = (root) => {
const contentTip = contentTipOf(root)
const best = nodes.get(contentTip) || nodes.get(root)
if (!best) return null
const latest = nodes.get(contentTip) || nodes.get(root)
if (!latest) return null
const rootAuthor = nodes.get(root) ? nodes.get(root).author : null
const groupKeys = []; { let x = root, g = 0; groupKeys.push(x); while (child.has(x) && g++ < 100000) { x = child.get(x); groupKeys.push(x) } }
const rootConfirms = confirmsByRoot.get(root) || []
const seal = rootConfirms.filter(cf => { const n = cf.tip ? nodes.get(cf.tip) : null; return n && n.author === rootAuthor && rootOf(cf.tip) === root && cf.author === n.c.to }).sort((a, b) => a.ts - b.ts)[0]
const best = seal ? nodes.get(seal.tip) : latest
const confirmedBy = new Set()
const opinions = {}; const opinionSet = new Set()
for (const k of groupKeys) {
const n = nodes.get(k); if (!n) continue; if (n.author !== rootAuthor) continue; const c = n.c
for (const cb of (Array.isArray(c.confirmedBy) ? c.confirmedBy : [])) confirmedBy.add(cb)
for (const v of (Array.isArray(c.opinions_inhabitants) ? c.opinions_inhabitants : [])) opinionSet.add(v)
if (c.opinions && typeof c.opinions === "object") for (const kk of Object.keys(c.opinions)) opinions[kk] = Math.max(opinions[kk] || 0, Number(c.opinions[kk]) || 0)
}
if (best.c.from) confirmedBy.add(best.c.from)
for (const cf of (confirmsByRoot.get(root) || [])) confirmedBy.add(cf.author)
if (best.c.from && best.c.from === best.author) confirmedBy.add(best.c.from)
for (const cf of rootConfirms) { if (cf.author === best.c.to) confirmedBy.add(cf.author) }
for (const op of (opinionsByRoot.get(root) || [])) { if (!opinionSet.has(op.author)) { opinionSet.add(op.author); opinions[op.category] = (opinions[op.category] || 0) + 1 } }
return { contentTip, best, confirmedBy: [...confirmedBy], opinions, opinions_inhabitants: [...opinionSet], tombstoned: tomb.has(contentTip) }
const ubiPayerTrusted = !!best.author && (best.author === defaultPub || !!(claimedPubs.get(best.c.to) && claimedPubs.get(best.c.to).has(best.author)))
return { contentTip, best, confirmedBy: [...confirmedBy], opinions, opinions_inhabitants: [...opinionSet], tombstoned: tomb.has(contentTip) || tomb.has(best.key), ubiPayerTrusted }
}
return { tomb, nodes, parent, child, rootOf, tipByRoot, resolveGroup }
}
const deriveStatus = (t) => {
const status = String(t.status || "").toUpperCase()
const isSettledUbi = (t) => {
const tags = Array.isArray(t.tags) ? t.tags.map(x => String(x).toUpperCase()) : []
return tags.includes("UBI") && !tags.includes("PENDING") && /^[0-9a-f]{64}$/i.test(String(t.txid || ""))
}
const deriveStatus = (t, ubiPayerTrusted = false) => {
const required = t.from === t.to ? 1 : 2
const confirmedCount = Array.isArray(t.confirmedBy) ? t.confirmedBy.length : 0
if (status === "DISCARDED") return "DISCARDED"
if (ubiPayerTrusted && isSettledUbi(t)) return "CLOSED"
if (confirmedCount >= required) return "CLOSED"
const dl = t.deadline ? moment(t.deadline) : null
if (dl && dl.isValid() && dl.isBefore(moment())) return "DISCARDED"
return status === "CLOSED" ? "CLOSED" : "UNCONFIRMED"
return "UNCONFIRMED"
}
const buildTransfer = (node, agg) => {
const c = node.c || {}
const confirmedBy = agg ? agg.confirmedBy : (Array.isArray(c.confirmedBy) ? c.confirmedBy : [])
const confirmedBy = agg ? agg.confirmedBy : (c.from && c.from === node.author ? [c.from] : [])
const opinions = agg ? agg.opinions : (c.opinions || {})
const opinions_inhabitants = agg ? agg.opinions_inhabitants : (Array.isArray(c.opinions_inhabitants) ? c.opinions_inhabitants : [])
return {
@ -203,8 +214,9 @@ module.exports = ({ cooler }) => {
updatedAt: c.updatedAt || null,
deadline: c.deadline,
confirmedBy,
status: deriveStatus({ ...c, confirmedBy }),
status: deriveStatus({ ...c, confirmedBy }, !!(agg && agg.ubiPayerTrusted)),
tags: Array.isArray(c.tags) ? c.tags : [],
txid: typeof c.txid === "string" ? c.txid : null,
opinions,
opinions_inhabitants
}
@ -269,6 +281,7 @@ module.exports = ({ cooler }) => {
const old = await getMsg(ssbClient, tipId)
if (!old?.content || old.content.type !== "transfer") throw new Error("Transfer not found")
if (old.author !== old.content.from) throw new Error("Not the author")
const current = old.content
const idxU = buildIndex(await getAllMessages(ssbClient))
@ -331,6 +344,7 @@ module.exports = ({ cooler }) => {
if (!msg?.content) throw new Error("Not found")
if (msg.content.type !== "transfer") throw new Error("Not found")
if (msg.author !== msg.content.from) throw new Error("Not found")
const idx = buildIndex(await getAllMessages(ssbClient))
const root = idx.rootOf(tipId)
@ -345,7 +359,7 @@ module.exports = ({ cooler }) => {
if ((Array.isArray(t.confirmedBy) ? t.confirmedBy : []).includes(userId)) throw new Error("Already confirmed")
const content = { type: "transferConfirm", target: root, createdAt: new Date().toISOString() }
const content = { type: "transferConfirm", target: root, tip: tipId, createdAt: new Date().toISOString() }
return new Promise((resolve, reject) => {
ssbClient.publish(content, (e2, result) => e2 ? reject(e2) : resolve(result))
})
@ -358,6 +372,7 @@ module.exports = ({ cooler }) => {
const msg = await getMsg(ssbClient, tipId)
if (!msg?.content || msg.content.type !== "transfer") throw new Error("Not found")
if (msg.author !== msg.content.from) throw new Error("Not the author")
const idxD = buildIndex(await getAllMessages(ssbClient))
const gD = idxD.resolveGroup(idxD.rootOf(tipId))
@ -396,7 +411,7 @@ module.exports = ({ cooler }) => {
async getTransferById(id) {
const ssbClient = await openSsb()
const messages = await getAllMessages(ssbClient)
const idx = buildIndex(messages)
const idx = memoIndex('transfers', messages, buildIndex)
const root = idx.rootOf(id)
const g = idx.resolveGroup(root)
@ -405,6 +420,7 @@ module.exports = ({ cooler }) => {
const msg = await getMsg(ssbClient, id)
if (!msg?.content || msg.content.type !== "transfer") throw new Error("Not found")
if (msg.author !== msg.content.from) throw new Error("Not found")
const tmpNode = { key: id, ts: msg.timestamp || 0, c: msg.content, author: msg.author }
return buildTransfer(tmpNode)

111
src/models/trending_model.js
View file

@ -3,7 +3,9 @@ const { getConfig } = require('../configs/config-manager.js');
const logLimit = getConfig().ssbLogStream?.limit || 1000;
const opinionCategories = require('../backend/opinion_categories');
const { buildValidatedTombstoneSet } = require('./tombstone_validator');
const { readTyped, CONTENT_TYPES, requestScope } = require('./typed_log');
const { buildVoteTally } = require('../backend/vote_tally');
const { versionGroups, versionEntry } = require('../backend/dedupe');
module.exports = ({ cooler }) => {
let ssb;
@ -20,27 +22,19 @@ module.exports = ({ cooler }) => {
const types = [
'bookmark', 'votes', 'poll', 'feed',
'image', 'audio', 'video', 'document', 'torrent', 'transfer',
'industry', 'project', 'report', 'task', 'event', 'shopProduct', 'housing', 'market'
'image', 'audio', 'video', 'document', 'torrent', 'file', 'transfer',
'industry', 'project', 'report', 'task', 'event', 'shopProduct', 'housing', 'market', 'schoolCourse',
'podcast', 'podcastEpisode', 'campaign', 'logisticsRoute'
];
const categories = opinionCategories;
const listTrending = async (filter = 'ALL') => {
const computeBase = async () => {
const ssbClient = await openSsb();
const userId = ssbClient.id;
const messages = await new Promise((res, rej) => {
pull(
ssbClient.createLogStream({ limit: logLimit }),
pull.collect((err, xs) => err ? rej(err) : res(xs))
);
});
const messages = await readTyped(ssbClient, CONTENT_TYPES, { limit: logLimit, withWindow: true });
const tombstoned = buildValidatedTombstoneSet(messages);
const replaces = new Map();
const itemsById = new Map();
const authorOf = new Map();
const opinionMsgs = [];
for (const m of messages) {
@ -48,54 +42,26 @@ module.exports = ({ cooler }) => {
const c = m.value?.content;
if (!c) continue;
if (c.type === 'tombstone' && c.target) {
tombstoned.add(c.target);
itemsById.delete(c.target);
continue;
}
if (k) authorOf.set(k, m.value?.author);
if (c.type === 'tombstone') continue;
if (typeof c.type === 'string' && c.type.endsWith('Opinion') && c.target) {
opinionMsgs.push({ target: c.target, author: m.value?.author, category: c.category });
continue;
}
if (c.opinions && !tombstoned.has(k) && !['task', 'event', 'report'].includes(c.type)) {
if (c.replaces) replaces.set(c.replaces, k);
itemsById.set(k, m);
}
if (c.opinions && !['task', 'event', 'report'].includes(c.type)) itemsById.set(k, m);
if (c.type === 'feed' && !tombstoned.has(k) && !itemsById.has(k)) {
if (c.replaces) replaces.set(c.replaces, k);
itemsById.set(k, m);
}
if (c.type === 'feed' && !itemsById.has(k)) itemsById.set(k, m);
}
for (const [replacedId, newId] of Array.from(replaces.entries())) {
const oldAuthor = authorOf.get(replacedId);
const newMsg = itemsById.get(newId);
if (oldAuthor === undefined) { replaces.delete(replacedId); continue; }
if (!newMsg || String(newMsg.value?.author) !== String(oldAuthor)) {
replaces.delete(replacedId);
itemsById.delete(newId);
continue;
}
itemsById.delete(replacedId);
const vg = versionGroups(Array.from(itemsById.values()).map(versionEntry));
for (const k of Array.from(itemsById.keys())) {
if (!vg.roots.has(vg.rootOf(k)) || vg.tipOf(k) !== k || tombstoned.has(k)) itemsById.delete(k);
}
const tipToChain = new Map();
for (const [oldId, newId] of replaces.entries()) {
let tip = newId; let g = 0;
while (replaces.has(tip) && g++ < 100000) tip = replaces.get(tip);
if (!tipToChain.has(tip)) tipToChain.set(tip, new Set([tip]));
tipToChain.get(tip).add(oldId); tipToChain.get(tip).add(newId);
}
const idToTip = new Map();
for (const [tip, chain] of tipToChain.entries()) for (const id of chain) idToTip.set(id, tip);
const opinionByTip = new Map();
for (const op of opinionMsgs) {
const tip = idToTip.get(op.target) || op.target;
const tip = vg.tipOf(op.target);
if (!opinionByTip.has(tip)) opinionByTip.set(tip, { opinions: {}, voters: new Set() });
const agg = opinionByTip.get(tip);
if (agg.voters.has(op.author)) continue;
@ -114,6 +80,12 @@ module.exports = ({ cooler }) => {
}
let rawItems = Array.from(itemsById.values()).filter(m => types.includes(m.value?.content?.type));
rawItems = rawItems.filter(m => {
const c = m.value?.content;
if (c?.type !== 'schoolCourse') return true;
if (c.encryptedPayload) return false;
return String(c.visibility || 'PUBLIC').toUpperCase() !== 'INVITE';
});
const blobTypes = ['document', 'image', 'audio', 'video'];
let items = await Promise.all(
@ -176,7 +148,21 @@ module.exports = ({ cooler }) => {
bySig.set(sig, m);
}
}
items = Array.from(bySig.values());
return Array.from(bySig.values());
};
const baseItems = () => {
const store = requestScope.getStore();
if (store && store.trendingBase) return store.trendingBase;
const pending = computeBase();
if (store) store.trendingBase = pending;
return pending;
};
const listTrending = async (filter = 'ALL') => {
const ssbClient = await openSsb();
const userId = ssbClient.id;
let items = (await baseItems()).slice();
if (filter === 'MINE') {
items = items.filter(m => m.value.author === userId);
@ -232,31 +218,12 @@ module.exports = ({ cooler }) => {
const inhabitants = Array.isArray(msg.content.opinions_inhabitants) ? msg.content.opinions_inhabitants : [];
if (inhabitants.includes(userId)) throw new Error('Already voted');
const otype = categories.messageTypes[type];
if (!otype) throw new Error('Voting not allowed on this content type');
const tombstone = {
type: 'tombstone',
target: contentId,
deletedAt: new Date().toISOString(),
author: userId
};
const updated = {
...msg.content,
opinions: {
...(msg.content.opinions || {}),
[category]: ((msg.content.opinions && msg.content.opinions[category]) || 0) + 1
},
opinions_inhabitants: inhabitants.concat(userId),
updatedAt: new Date().toISOString(),
replaces: contentId
};
await new Promise((res, rej) => {
ssbClient.publish(tombstone, err => err ? rej(err) : res());
});
const content = { type: otype, target: contentId, category, createdAt: new Date().toISOString() };
return new Promise((res, rej) => {
ssbClient.publish(updated, (err, result) => err ? rej(err) : res(result));
ssbClient.publish(content, (err, result) => err ? rej(err) : res(result));
});
};

1
src/models/tribe_crypto.js
View file

@ -1 +0,0 @@
module.exports = (configPath) => require('./crypto')(configPath, 'tribes');

205
src/models/tribes_content_model.js
View file

@ -2,11 +2,30 @@ const pull = require('../server/node_modules/pull-stream');
const { getConfig } = require('../configs/config-manager.js');
const logLimit = getConfig().ssbLogStream?.limit || 1000;
const tribeLogLimit = Math.max(logLimit, 100000);
const { readTyped } = require('./typed_log');
const longText = require('../backend/long_text');
const { versionGroups } = require('../backend/dedupe');
const TRIBE_LOG_TYPES = ['tribe', 'tribe-msg', 'tribe-keys-distrib', 'tribe-invite-msg', 'tribe-invite-tombstone', 'tribe-open-invite-tombstone', 'tombstone', 'contact'];
const VALID_CONTENT_TYPES = ['event', 'task', 'report', 'votation', 'forum', 'forum-reply', 'market', 'job', 'project', 'media', 'feed', 'pixelia'];
const categories = require('../backend/opinion_categories');
const VALID_STATUSES = ['OPEN', 'CLOSED', 'IN-PROGRESS'];
const VALID_PRIORITIES = ['LOW', 'MEDIUM', 'HIGH', 'CRITICAL'];
const EXPOSURE_TYPE = 'tribeExposure';
const REACH_LEVELS = ['tribe', 'oasis', 'clearnet'];
const REACH_RANK = { tribe: 0, oasis: 1, clearnet: 2 };
const SNAPSHOT_FIELDS = ['contentType', 'title', 'description', 'status', 'date', 'location', 'price', 'salary', 'priority', 'options', 'category', 'tags', 'image', 'mediaType', 'license', 'url', 'deadline', 'goal', 'dates', 'lat', 'lng', 'markers', 'createdAt', 'updatedAt'];
const snapshotOf = (item, replies = []) => {
const out = {};
for (const f of SNAPSHOT_FIELDS) {
const v = item[f];
if (v === undefined || v === null || v === '' || (Array.isArray(v) && !v.length)) continue;
out[f] = v;
}
out.author = item.author;
if (item.contentType === 'forum') out.replies = (replies || []).map(r => ({ author: r.author, description: r.description || '', createdAt: r.createdAt || null }));
return out;
};
module.exports = ({ cooler, tribeCrypto, tribesModel }) => {
let ssb;
@ -14,12 +33,7 @@ module.exports = ({ cooler, tribeCrypto, tribesModel }) => {
const readLog = async () => {
const client = await openSsb();
return new Promise((resolve, reject) =>
pull(
client.createLogStream({ limit: tribeLogLimit }),
pull.collect((err, msgs) => err ? reject(err) : resolve(msgs))
)
);
return readTyped(client, TRIBE_LOG_TYPES, { limit: tribeLogLimit });
};
const fingerprintsForRoot = (rootId) => {
@ -38,6 +52,12 @@ module.exports = ({ cooler, tribeCrypto, tribesModel }) => {
);
};
const publishContent = async (rootId, body) => {
const publishChunk = (c) => wrapAndPublishContent(rootId, { k: longText.CHUNK_TYPE, rootId, text: c.text, index: c.index, total: c.total, group: c.group });
const sealed = await longText.chunkContent(body, 'description', { maxBytes: longText.CONTENT_BYTES.sealed, extra: { rootId }, publish: publishChunk });
return wrapAndPublishContent(rootId, sealed);
};
const decodeContentMsgs = async (msgs, opts) => {
const fpIdx = tribeCrypto.buildFingerprintIndex();
const targetRootId = opts && opts.rootId ? opts.rootId : null;
@ -45,6 +65,7 @@ module.exports = ({ cooler, tribeCrypto, tribesModel }) => {
const allowedFps = targetRootId ? fingerprintsForRoot(targetRootId) : null;
const content = new Map();
const chunks = new Map();
const tombRequests = [];
const collabMsgs = [];
@ -62,36 +83,30 @@ module.exports = ({ cooler, tribeCrypto, tribesModel }) => {
tombRequests.push({ target: b.target, author: m.value.author });
continue;
}
if (b.k === longText.CHUNK_TYPE) {
const ch = longText.chunkOf(m.value.author, b);
if (ch) chunks.set(m.key, ch);
continue;
}
if (b.k === 'tc-collab') {
collabMsgs.push({ sub: b.sub, target: b.target, author: m.value.author, ts: m.value.timestamp || 0, option: b.option, on: b.on, category: b.category });
collabMsgs.push({ sub: b.sub, target: b.target, author: m.value.author, ts: m.value.timestamp || 0, seq: m.value.sequence || 0, option: b.option, on: b.on, category: b.category });
continue;
}
if (b.k !== 'tribe-content') continue;
content.set(m.key, { author: m.value.author, body: b, ts: m.value.timestamp });
content.set(m.key, { author: m.value.author, body: b, ts: m.value.timestamp, seq: m.value.sequence || 0 });
}
const naiveNext = new Map();
const strictNext = new Map();
for (const [key, node] of content) {
const t = node.body.replaces;
if (!t) continue;
naiveNext.set(t, key);
const orig = content.get(t);
if (orig && orig.author === node.author) strictNext.set(t, key);
if (targetRootId && tribesModel && typeof tribesModel.departures === 'function') {
const gone = await tribesModel.departures(targetRootId).catch(() => new Map());
const afterLeaving = (author, ts, seq) => {
const d = gone.get(author);
return !!d && ((d.seq !== undefined && seq > d.seq) || (d.ts !== undefined && (Number(ts) || 0) > d.ts));
};
for (const [key, node] of [...content]) if (afterLeaving(node.author, node.ts, node.seq)) content.delete(key);
for (let i = collabMsgs.length - 1; i >= 0; i--) if (afterLeaving(collabMsgs[i].author, collabMsgs[i].ts, collabMsgs[i].seq)) collabMsgs.splice(i, 1);
}
for (const [t, key] of Array.from(naiveNext.entries())) {
const orig = content.get(t);
if (!orig) { naiveNext.delete(t); continue; }
const node = content.get(key);
if (!node || node.author !== orig.author) { naiveNext.delete(t); content.delete(key); }
}
const naivePrev = new Map();
for (const [t, key] of naiveNext) naivePrev.set(key, t);
const followTo = (start, nextMap) => { let x = start, g = 0; while (nextMap.has(x) && g++ < 100000) x = nextMap.get(x); return x; };
const rootOf = (key) => { let x = key, g = 0; while (naivePrev.has(x) && content.has(naivePrev.get(x)) && g++ < 100000) x = naivePrev.get(x); return x; };
const roots = new Set();
for (const [key, node] of content) { const t = node.body.replaces; if (!t || !content.has(t)) roots.add(key); }
const vg = versionGroups([...content].map(([key, node]) => ({ key, author: node.author, seq: node.seq, ts: node.ts, replaces: typeof node.body.replaces === 'string' ? node.body.replaces : null, claimedAuthor: typeof node.body.author === 'string' ? node.body.author : null })));
const tombstoned = new Set();
for (const t of tombRequests) { const orig = content.get(t.target); if (orig && t.author === orig.author) tombstoned.add(t.target); }
@ -100,7 +115,7 @@ module.exports = ({ cooler, tribeCrypto, tribesModel }) => {
const getCB = (root) => { if (!collabByRoot.has(root)) collabByRoot.set(root, { vote: new Map(), attend: new Map(), assign: new Map(), refeed: new Set(), opinion: new Map() }); return collabByRoot.get(root); };
for (const cm of collabMsgs) {
if (!content.has(cm.target)) continue;
const a = getCB(rootOf(cm.target));
const a = getCB(vg.rootOf(cm.target));
if (cm.sub === 'vote' && Number.isInteger(cm.option)) { if (!a.vote.has(cm.author)) a.vote.set(cm.author, cm.option); }
else if (cm.sub === 'attend') { const p = a.attend.get(cm.author); if (!p || cm.ts >= p.ts) a.attend.set(cm.author, { on: cm.on !== false, ts: cm.ts }); }
else if (cm.sub === 'assign') { const p = a.assign.get(cm.author); if (!p || cm.ts >= p.ts) a.assign.set(cm.author, { on: cm.on !== false, ts: cm.ts }); }
@ -112,15 +127,15 @@ module.exports = ({ cooler, tribeCrypto, tribesModel }) => {
const tipOf = new Map();
const EMPTY = { vote: new Map(), attend: new Map(), assign: new Map(), refeed: new Set(), opinion: new Map() };
for (const root of roots) {
const contentTip = followTo(root, strictNext);
for (const root of vg.roots) {
const contentTip = vg.tipOf(root);
const tipNode = content.get(contentTip);
if (!tipNode) continue;
const body = tipNode.body;
const lb = tipNode.body;
const a = collabByRoot.get(root) || EMPTY;
const item = { id: contentTip, ...body, author: tipNode.author, _ts: tipNode.ts };
const item = { id: contentTip, ...body, description: longText.joinText(body.description, body.chunks, longText.lookupIn(chunks), tipNode.author), author: tipNode.author, _ts: tipNode.ts, originId: root };
const voteAuthors = new Set(); const votes = {};
const legacyVotes = lb.votes && typeof lb.votes === 'object' ? lb.votes : {};
@ -145,8 +160,7 @@ module.exports = ({ cooler, tribeCrypto, tribesModel }) => {
for (const [au, cat] of a.opinion) if (!opInh.has(au)) { opInh.add(au); opinions[cat] = (opinions[cat] || 0) + 1; }
item.opinions_inhabitants = [...opInh]; item.opinions = opinions;
let cur = root, g = 0; tipOf.set(root, contentTip);
while (naiveNext.has(cur) && g++ < 100000) { cur = naiveNext.get(cur); tipOf.set(cur, contentTip); }
for (const k of (vg.groups.get(root) || [root])) tipOf.set(k, contentTip);
if (tombstoned.has(contentTip)) continue;
if (targetRootId && body.rootId !== targetRootId) continue;
@ -157,7 +171,23 @@ module.exports = ({ cooler, tribeCrypto, tribesModel }) => {
return { items, tipOf, tombstoned };
};
const chainIsPublic = async (tribeId) => {
let cur;
try { cur = await tribesModel.getTribeById(tribeId); } catch (_) { return false; }
const seen = new Set();
while (cur) {
if (cur.isAnonymous) return false;
if (!cur.parentTribeId) return true;
if (seen.has(cur.parentTribeId)) return false;
seen.add(cur.parentTribeId);
try { cur = await tribesModel.getTribeById(cur.parentTribeId); } catch (_) { return false; }
}
return false;
};
return {
chainIsPublic,
async create(tribeId, contentType, data) {
if (!VALID_CONTENT_TYPES.includes(contentType)) throw new Error('Invalid content type');
if (data.status && !VALID_STATUSES.includes(data.status)) throw new Error('Invalid status. Must be OPEN, CLOSED, or IN-PROGRESS');
@ -187,6 +217,7 @@ module.exports = ({ cooler, tribeCrypto, tribesModel }) => {
tags: data.tags || [],
image: data.image || null,
mediaType: data.mediaType || null,
...(data.license ? { license: data.license } : {}),
url: data.url || null,
attendees: data.attendees || [],
deadline: data.deadline || null,
@ -200,7 +231,7 @@ module.exports = ({ cooler, tribeCrypto, tribesModel }) => {
createdAt: now,
updatedAt: now
};
return wrapAndPublishContent(rootId, body);
return publishContent(rootId, body);
},
async update(contentId, data, existing) {
@ -215,7 +246,7 @@ module.exports = ({ cooler, tribeCrypto, tribesModel }) => {
k: 'tribe-content',
rootId,
contentType: existing.contentType,
replaces: contentId,
replaces: existing.id || contentId,
title: data.title !== undefined ? data.title : existing.title,
description: data.description !== undefined ? data.description : existing.description,
status: data.status !== undefined ? data.status : existing.status,
@ -232,6 +263,7 @@ module.exports = ({ cooler, tribeCrypto, tribesModel }) => {
tags: data.tags !== undefined ? data.tags : existing.tags,
image: data.image !== undefined ? data.image : existing.image,
mediaType: data.mediaType !== undefined ? data.mediaType : existing.mediaType,
...((data.license !== undefined ? data.license : existing.license) ? { license: data.license !== undefined ? data.license : existing.license } : {}),
url: data.url !== undefined ? data.url : existing.url,
attendees: data.attendees !== undefined ? data.attendees : existing.attendees,
deadline: data.deadline !== undefined ? data.deadline : existing.deadline,
@ -245,7 +277,7 @@ module.exports = ({ cooler, tribeCrypto, tribesModel }) => {
createdAt: existing.createdAt,
updatedAt: now
};
return wrapAndPublishContent(rootId, body);
return publishContent(rootId, body);
},
async deleteById(contentId) {
@ -256,7 +288,7 @@ module.exports = ({ cooler, tribeCrypto, tribesModel }) => {
const body = {
k: 'tombstone',
rootId,
target: contentId,
target: existing.id || contentId,
author: client.id,
deletedAt: new Date().toISOString()
};
@ -347,6 +379,103 @@ module.exports = ({ cooler, tribeCrypto, tribesModel }) => {
return wrapAndPublishContent(item.rootId, { k: 'tc-collab', sub: 'opinion', rootId: item.rootId, target: item.id, category });
},
reachLevels: REACH_LEVELS,
async publishExposure(item, level, replies = []) {
if (!REACH_LEVELS.includes(level)) throw new Error('Invalid level');
if (!item || !item.rootId) throw new Error('Content not found');
const client = await openSsb();
const publish = (c) => new Promise((resolve, reject) => client.publish(c, (err, r) => err ? reject(err) : resolve(r)));
const content = {
type: EXPOSURE_TYPE,
tribeId: item.rootId,
origin: item.originId || item.id,
contentType: item.contentType,
level,
item: null,
at: new Date().toISOString()
};
if (level !== 'tribe') {
const { replies: snapReplies, ...snapshot } = snapshotOf(item, replies);
const budget = longText.CONTENT_BYTES.plain - longText.contentBytes(content);
const sealed = await longText.chunkContent(snapshot, 'description', { maxBytes: budget, publish });
if (Array.isArray(snapReplies)) {
sealed.replies = snapReplies.slice();
while (sealed.replies.length && longText.contentBytes({ ...content, item: sealed }) > longText.CONTENT_BYTES.plain) sealed.replies.pop();
}
content.item = sealed;
}
return publish(content);
},
async exposureState({ ignorePrivacy = false } = {}) {
const client = await openSsb();
const msgs = await readTyped(client, [EXPOSURE_TYPE, longText.CHUNK_TYPE], { limit: tribeLogLimit });
const lookup = longText.lookupIn(longText.indexChunks(msgs));
const byOrigin = new Map();
for (const m of msgs) {
const c = m && m.value && m.value.content;
if (!c || c.type !== EXPOSURE_TYPE || typeof c.origin !== 'string' || typeof c.tribeId !== 'string' || !REACH_LEVELS.includes(c.level)) continue;
if (!byOrigin.has(c.origin)) byOrigin.set(c.origin, []);
byOrigin.get(c.origin).push({ by: m.value.author, ts: Number(m.value.timestamp) || 0, c });
}
const memberCache = new Map();
const isMember = async (author, tribeId) => {
const k = `${tribeId}|${author}`;
if (!memberCache.has(k)) memberCache.set(k, await tribesModel.isTribeMember(author, tribeId).catch(() => false));
return memberCache.get(k);
};
const everCache = new Map();
const wasMember = async (author, tribeId) => {
const k = `${tribeId}|${author}`;
if (!everCache.has(k)) everCache.set(k, await tribesModel.wasEverMember(author, tribeId).catch(() => false));
return everCache.get(k);
};
const privacyCache = new Map();
const isPrivate = async (tribeId) => {
if (!privacyCache.has(tribeId)) privacyCache.set(tribeId, !(await chainIsPublic(tribeId)));
return privacyCache.get(tribeId);
};
const state = new Map();
for (const [origin, list] of byOrigin) {
list.sort((a, b) => (b.ts - a.ts) || (REACH_RANK[a.c.level] - REACH_RANK[b.c.level]));
let chosen = null;
for (const e of list) {
if (await isMember(e.by, e.c.tribeId)) { chosen = e; break; }
if (e.c.level === 'tribe' && await wasMember(e.by, e.c.tribeId)) { chosen = e; break; }
}
if (!chosen) continue;
const priv = await isPrivate(chosen.c.tribeId);
const level = !ignorePrivacy && priv ? 'tribe' : chosen.c.level;
state.set(origin, { origin, tribeId: chosen.c.tribeId, contentType: chosen.c.contentType, level, item: level === 'tribe' || !chosen.c.item ? null : longText.resolveField(chosen.c.item, 'description', chosen.by, lookup), by: chosen.by, ts: chosen.ts });
}
return state;
},
async levelOf(item, state = null) {
const st = state || await this.exposureState();
const e = st.get(item && (item.originId || item.id));
return e ? e.level : 'tribe';
},
async listExposed(tribeId, contentType = null, minLevel = 'oasis', state = null) {
const rootId = await tribesModel.getRootId(tribeId).catch(() => tribeId);
const st = state || await this.exposureState();
const out = [];
for (const e of st.values()) {
if (e.tribeId !== rootId || !e.item || REACH_RANK[e.level] < REACH_RANK[minLevel]) continue;
const base = { ...e.item, id: e.origin, originId: e.origin, rootId, reach: e.level, exposedBy: e.by, _ts: e.ts, attendees: [], assignees: [], votes: {}, opinions: {}, opinions_inhabitants: [], refeeds: 0, refeeds_inhabitants: [] };
if (contentType === 'forum-reply') {
if (e.item.contentType !== 'forum') continue;
(e.item.replies || []).forEach((r, i) => out.push({ id: `${e.origin}:${i}`, originId: `${e.origin}:${i}`, rootId, contentType: 'forum-reply', parentId: e.origin, description: r.description, author: r.author, createdAt: r.createdAt, reach: e.level, attendees: [], assignees: [], votes: {}, opinions: {}, opinions_inhabitants: [], refeeds_inhabitants: [] }));
continue;
}
if (contentType && e.item.contentType !== contentType) continue;
out.push(base);
}
return out.sort((a, b) => (Date.parse(b.updatedAt || b.createdAt) || b._ts || 0) - (Date.parse(a.updatedAt || a.createdAt) || a._ts || 0));
},
async getThread(forumId) {
const msgs = await readLog();
const { items } = await decodeContentMsgs(msgs, {});

461
src/models/tribes_model.js
View file

@ -4,6 +4,8 @@ const { getConfig } = require('../configs/config-manager.js');
const { buildValidatedTombstoneSet } = require('./tombstone_validator');
const logLimit = getConfig().ssbLogStream?.limit || 1000;
const tribeLogLimit = Math.max(logLimit, 100000);
const { readTyped } = require('./typed_log');
const TRIBE_LOG_TYPES = ['tribe', 'tribe-msg', 'tribe-keys-distrib', 'tribe-invite-msg', 'tribe-invite-tombstone', 'tribe-open-invite', 'tribe-open-invite-tombstone', 'tombstone', 'contact'];
const INVITE_CODE_BYTES = 16;
const VALID_INVITE_MODES = ['strict', 'open'];
@ -25,13 +27,13 @@ module.exports = ({ cooler, tribeCrypto }) => {
return true;
};
const validMembershipDelta = (prev, next, author) => {
const validMembershipDelta = (prev, next, author, selfJoinOk = () => true) => {
const a = Array.isArray(prev) ? prev : [];
const b = Array.isArray(next) ? next : [];
const added = b.filter(m => !a.includes(m));
const removed = a.filter(m => !b.includes(m));
if (added.length === 0 && removed.length === 0) return true;
if (added.length === 1 && removed.length === 0 && added[0] === author) return true;
if (added.length === 1 && removed.length === 0 && added[0] === author) return selfJoinOk() === true;
if (removed.length === 1 && added.length === 0 && removed[0] === author) return true;
return false;
};
@ -49,19 +51,29 @@ module.exports = ({ cooler, tribeCrypto }) => {
const x = a[f];
const y = b[f];
if (Array.isArray(x) || Array.isArray(y)) { if (!arraysEqual(x, y)) return false; continue; }
if (x !== y && !(x == null && y == null)) return false;
if (x !== y && !((x == null || x === '') && (y == null || y === ''))) return false;
}
return true;
};
const streamLog = async () => {
const client = await openSsb();
return new Promise((resolve, reject) => {
pull(
client.createLogStream({ limit: tribeLogLimit }),
pull.collect((err, msgs) => err ? reject(err) : resolve(msgs))
);
});
return readTyped(client, TRIBE_LOG_TYPES, { limit: tribeLogLimit });
};
const readKeyDistribs = async () => {
const client = await openSsb();
const ssbKeys = require('../server/node_modules/ssb-keys');
const config = require('../server/ssb_config');
const msgs = await readTyped(client, TRIBE_LOG_TYPES, { limit: tribeLogLimit, withPrivate: true });
const out = [];
for (const m of msgs) {
const v = m && m.value;
if (!v) continue;
const c = v.content;
const dec = (c && typeof c === 'object') ? (v.private === true && tribeCrypto.isKeyDistribContent(c) ? c : null) : tribeCrypto.tryUnboxKeyDistrib(c, config.keys, ssbKeys);
if (dec) out.push({ m, dec });
}
return { msgs, distribs: out };
};
const buildTribeIndex = async () => {
@ -72,17 +84,31 @@ module.exports = ({ cooler, tribeCrypto }) => {
const tribeMsgs = new Map();
const tombstones = new Map();
const inviteAuthorByCh = new Map();
const inviteAuthorByKey = new Map();
const inviteTombstonesByAuthor = new Map();
const sealedInvites = [];
for (const m of msgs) {
const c = m.value && m.value.content;
if (!c) continue;
const author = m.value.author;
const ts = m.value.timestamp;
if (c.type === 'tribe-invite-msg' && typeof c.ch === 'string') { inviteAuthorByCh.set(c.ch, author); inviteAuthorByKey.set(m.key, author); continue; }
if (c.type === 'tribe-invite-tombstone' && typeof c.target === 'string') {
if (!inviteTombstonesByAuthor.has(author)) inviteTombstonesByAuthor.set(author, new Set());
inviteTombstonesByAuthor.get(author).add(c.target);
continue;
}
let body = null;
if (tribeCrypto.isTribeMsg(c)) {
const r = tribeCrypto.unwrapMsg(c, fpIdx);
if (!r || !r.body || !r.body.k) continue;
if (r.body.k === 'open-invite' || r.body.k === 'open-invite-tombstone') {
sealedInvites.push({ key: m.key, author, body: r.body, keyRoot: r.rootId, ts });
continue;
}
body = r.body;
} else if (c.type === 'tribe' && typeof c === 'object') {
body = {
@ -104,7 +130,9 @@ module.exports = ({ cooler, tribeCrypto }) => {
mapUrl: c.mapUrl,
createdAt: c.createdAt,
updatedAt: c.updatedAt,
author: c.author
author: c.author,
joinedWith: typeof c.joinedWith === 'string' ? c.joinedWith : undefined,
removedSeqs: c.removedSeqs && typeof c.removedSeqs === 'object' ? c.removedSeqs : undefined
};
} else if (c.type === 'tombstone' && c.target) {
tombstones.set(c.target, { author, ts });
@ -125,6 +153,7 @@ module.exports = ({ cooler, tribeCrypto }) => {
op: body.op || (body.rootId ? 'update' : 'create'),
content: body,
author,
seq: (m.value && m.value.sequence) || 0,
_ts: ts
});
}
@ -142,49 +171,58 @@ module.exports = ({ cooler, tribeCrypto }) => {
}
}
let progress = true;
while (progress) {
progress = false;
const candidatesByReplaces = new Map();
for (const [k, entry] of tribeMsgs.entries()) {
const byReplaces = new Map();
for (const [k, entry] of tribeMsgs.entries()) {
if (!entry.replaces) continue;
if (!byReplaces.has(entry.replaces)) byReplaces.set(entry.replaces, []);
byReplaces.get(entry.replaces).push([k, entry]);
}
const queue = [...tribes.keys()];
for (let qi = 0; qi < queue.length; qi++) {
const replaces = queue[qi];
if (child.has(replaces)) continue;
const parentEntry = tribes.get(replaces);
const root = rootByTip.get(replaces);
const rootEntry = tribes.get(root);
const rootAuthor = rootEntry && rootEntry.author;
const prevMembers = Array.isArray(parentEntry.content.members) ? parentEntry.content.members : [];
const candidates = [];
for (const [k, entry] of (byReplaces.get(replaces) || [])) {
if (tribes.has(k)) continue;
const replaces = entry.replaces;
if (!replaces) continue;
const parentEntry = tribes.get(replaces);
if (!parentEntry) continue;
if (child.has(replaces)) continue;
const root = rootByTip.get(replaces);
const rootEntry = tribes.get(root);
const rootAuthor = rootEntry && rootEntry.author;
const isRootAuthor = entry.author === rootAuthor;
const prevMembers = Array.isArray(parentEntry.content.members) ? parentEntry.content.members : [];
if (!isRootAuthor) {
if (!prevMembers.includes(entry.author) && !(entry.content.members || []).includes(entry.author)) continue;
if (!validMembershipDelta(prevMembers, entry.content.members, entry.author)) continue;
const mode = String(parentEntry.content.inviteMode || (rootEntry && rootEntry.content.inviteMode) || 'strict');
const inviterOk = (inviter) => !!inviter && (inviter === rootAuthor || (mode === 'open' && prevMembers.includes(inviter)));
const selfJoinOk = () => {
const proof = entry.content.joinedWith;
if (proof && inviterOk(inviteAuthorByCh.get(proof))) return true;
const mine = inviteTombstonesByAuthor.get(entry.author);
if (mine) for (const target of mine) if (inviterOk(inviteAuthorByKey.get(target))) return true;
return false;
};
if (!validMembershipDelta(prevMembers, entry.content.members, entry.author, selfJoinOk)) continue;
if (!validInvitesDelta(parentEntry.content.invites, entry.content.invites, entry.author, rootAuthor)) continue;
if (!structuralFieldsEqual(parentEntry.content, entry.content)) continue;
}
if (!candidatesByReplaces.has(replaces)) candidatesByReplaces.set(replaces, []);
candidatesByReplaces.get(replaces).push({ k, entry, isRootAuthor, root });
candidates.push({ k, entry, isRootAuthor, root });
}
for (const [replaces, candidates] of candidatesByReplaces.entries()) {
if (child.has(replaces)) continue;
let winner = candidates[0];
for (let i = 1; i < candidates.length; i++) {
const c = candidates[i];
if (c.isRootAuthor && !winner.isRootAuthor) { winner = c; continue; }
if (winner.isRootAuthor && !c.isRootAuthor) continue;
const wt = winner.entry._ts || 0;
const ct = c.entry._ts || 0;
if (ct < wt) winner = c;
else if (ct === wt && c.k < winner.k) winner = c;
}
parent.set(winner.k, replaces);
child.set(replaces, winner.k);
tribes.set(winner.k, winner.entry);
rootByTip.set(winner.k, winner.root);
progress = true;
if (!candidates.length) continue;
let winner = candidates[0];
for (let i = 1; i < candidates.length; i++) {
const c = candidates[i];
if (c.isRootAuthor && !winner.isRootAuthor) { winner = c; continue; }
if (winner.isRootAuthor && !c.isRootAuthor) continue;
const wt = winner.entry._ts || 0;
const ct = c.entry._ts || 0;
if (ct < wt) winner = c;
else if (ct === wt && c.k < winner.k) winner = c;
}
parent.set(winner.k, replaces);
child.set(replaces, winner.k);
tribes.set(winner.k, winner.entry);
rootByTip.set(winner.k, winner.root);
queue.push(winner.k);
}
const tombstoned = new Set();
@ -197,37 +235,34 @@ module.exports = ({ cooler, tribeCrypto }) => {
}
const rootOf = (id) => rootByTip.get(id) || id;
const tipOf = (id) => { let cur = id; while (child.has(cur)) cur = child.get(cur); return cur; };
const tipByRoot = new Map();
for (const k of tribes.keys()) {
const root = rootOf(k);
const tip = tipOf(root);
if (tipByRoot.has(root)) continue;
let tip = root;
while (child.has(tip)) tip = child.get(tip);
tipByRoot.set(root, tip);
}
const effectivelyTombstoned = new Set(tombstoned);
let cascade = true;
const seen = new Set();
while (cascade) {
cascade = false;
for (const k of tribes.keys()) {
if (effectivelyTombstoned.has(k)) continue;
const root = rootOf(k);
if (effectivelyTombstoned.has(root)) { effectivelyTombstoned.add(k); cascade = true; continue; }
const deadMemo = new Map();
const isDead = (k) => {
if (tombstoned.has(k)) return true;
if (deadMemo.has(k)) return deadMemo.get(k);
deadMemo.set(k, false);
const root = rootOf(k);
let dead = tombstoned.has(tipByRoot.get(root)) || (root !== k && isDead(root));
if (!dead) {
const e = tribes.get(k);
const pid = e && e.content && e.content.parentTribeId;
if (!pid) continue;
if (seen.has(`${k}:${pid}`)) continue;
seen.add(`${k}:${pid}`);
const parentRoot = rootOf(pid);
if (effectivelyTombstoned.has(parentRoot) || effectivelyTombstoned.has(pid)) {
effectivelyTombstoned.add(k);
cascade = true;
}
if (pid) dead = isDead(rootOf(pid)) || isDead(pid);
}
}
deadMemo.set(k, dead);
return dead;
};
for (const k of tribes.keys()) if (isDead(k)) effectivelyTombstoned.add(k);
tribeIndex = { tribes, tombstoned, effectivelyTombstoned, parent, child, tipByRoot, rootByTip };
tribeIndex = { tribes, tombstoned, effectivelyTombstoned, parent, child, tipByRoot, rootByTip, sealedInvites };
tribeIndexTs = Date.now();
return tribeIndex;
};
@ -253,8 +288,9 @@ module.exports = ({ cooler, tribeCrypto }) => {
};
})();
const normalizeTribe = (entry) => {
const normalizeTribe = (entry, rootEntry) => {
const c = entry.content;
const root = rootEntry || entry;
return {
id: entry.id,
title: c.title || '',
@ -269,9 +305,9 @@ module.exports = ({ cooler, tribeCrypto }) => {
status: c.status || 'OPEN',
parentTribeId: c.parentTribeId || null,
mapUrl: c.mapUrl || '',
createdAt: c.createdAt,
createdAt: root.content.createdAt,
updatedAt: c.updatedAt,
author: c.author || entry.author,
author: root.author,
_ts: entry._ts
};
};
@ -290,28 +326,95 @@ module.exports = ({ cooler, tribeCrypto }) => {
const scanOpenInviteMarkers = async () => {
const msgs = await streamLog();
const tombstoned = new Set();
for (const m of msgs) {
const c = m.value && m.value.content;
if (!c) continue;
if (c.type === 'tribe-open-invite-tombstone' && typeof c.target === 'string') tombstoned.add(c.target);
if (c.type === 'tribe-invite-tombstone' && typeof c.target === 'string') tombstoned.add('inv:' + c.target);
}
const byRoot = new Map();
const idx = await buildTribeIndex();
const tombBy = new Map();
const markTomb = (target, author) => {
if (!tombBy.has(target)) tombBy.set(target, new Set());
tombBy.get(target).add(author);
};
const markers = [];
for (const m of msgs) {
const v = m.value;
const c = v && v.content;
if (!c || c.type !== 'tribe-open-invite' || c.v !== 1) continue;
if (!c) continue;
if (c.type === 'tribe-open-invite-tombstone' && typeof c.target === 'string') markTomb(c.target, v.author);
else if (c.type === 'tribe-invite-tombstone' && typeof c.target === 'string') markTomb('inv:' + c.target, v.author);
else if (c.type === 'tribe-open-invite' && c.v === 1) markers.push({ key: m.key, author: v.author, body: c, sealed: false, ts: v.timestamp || 0 });
}
for (const e of idx.sealedInvites || []) {
if (e.body.k === 'open-invite-tombstone' && typeof e.body.target === 'string') markTomb(e.body.target, e.author);
else if (e.body.k === 'open-invite' && e.body.rootId === e.keyRoot) markers.push({ key: e.key, author: e.author, body: e.body, sealed: true, ts: e.ts || 0 });
}
const killed = (target, allowed) => {
const by = tombBy.get(target);
return !!by && allowed.some((a) => a && by.has(a));
};
const byRoot = new Map();
for (const mk of markers) {
const c = mk.body;
if (typeof c.rootId !== 'string' || typeof c.code !== 'string') continue;
if (tombstoned.has(m.key)) continue;
if (c.inviteKey && tombstoned.has('inv:' + c.inviteKey)) continue;
const ts = (v && v.timestamp) || 0;
const rootEntry = idx.tribes.get(c.rootId);
if (!rootEntry || !mk.author) continue;
const tip = idx.tribes.get(idx.tipByRoot.get(c.rootId) || c.rootId) || rootEntry;
if (tip.content.isAnonymous !== false && !mk.sealed) continue;
const members = Array.isArray(tip.content.members) ? tip.content.members : [];
if (mk.author !== rootEntry.author && !members.includes(mk.author)) continue;
const allowed = [mk.author, rootEntry.author];
if (killed(mk.key, allowed)) continue;
if (c.inviteKey && killed('inv:' + c.inviteKey, allowed)) continue;
const prev = byRoot.get(c.rootId);
if (!prev || ts > prev.ts) byRoot.set(c.rootId, { code: c.code, by: c.by || v.author, markerKey: m.key, inviteKey: c.inviteKey || null, ts });
if (!prev || mk.ts > prev.ts) byRoot.set(c.rootId, { code: c.code, by: c.by || mk.author, markerKey: mk.key, inviteKey: c.inviteKey || null, sealed: mk.sealed, ts: mk.ts });
}
return byRoot;
};
const findInvite = async (code) => {
const msgs = await streamLog();
const inviteTombstonedBy = new Map();
const clearOpenInvites = new Set();
for (const m of msgs) {
const v = m.value;
const c = v && v.content;
if (!c) continue;
if (c.type === 'tribe-open-invite' && typeof c.inviteKey === 'string') { clearOpenInvites.add(c.inviteKey); continue; }
if (c.type !== 'tribe-invite-tombstone' || typeof c.target !== 'string') continue;
if (!inviteTombstonedBy.has(c.target)) inviteTombstonedBy.set(c.target, new Set());
inviteTombstonedBy.get(c.target).add(v.author);
}
let matched = null;
for (const m of msgs) {
const v = m.value;
const c = v && v.content;
if (!c || c.type !== 'tribe-invite-msg' || c.v !== 1) continue;
if (typeof c.ch !== 'string' || typeof c.s !== 'string' || typeof c.ek !== 'string') continue;
const multi = c.multi === 1 || c.multi === true;
if (!multi && inviteTombstonedBy.has(m.key)) continue;
if (tribeCrypto.hashInviteCode(code, c.s) !== c.ch) continue;
const chain = tribeCrypto.decryptChainFromInvite(c.ek, code, c.s, 3);
if (Array.isArray(chain) && chain.length) {
matched = { msgKey: m.key, codeHash: c.ch, chain, multi, inviteAuthor: v.author };
break;
}
}
if (!matched) return null;
const fresh = matched.chain.filter(entry => entry && Array.isArray(entry.keys) && entry.keys.length && !tribeCrypto.getKeys(entry.rootId).length);
for (const entry of fresh) tribeCrypto.setKeys(entry.rootId, entry.keys, entry.gen || entry.keys.length);
tribeIndex = null;
const undo = () => { for (const entry of fresh) tribeCrypto.dropKey(entry.rootId); tribeIndex = null; };
const rootId = matched.chain[0].rootId;
const idx = await buildTribeIndex().catch(() => null);
const entry = idx && idx.tribes.get(idx.tipByRoot.get(rootId) || rootId);
const tribe = entry && !idx.effectivelyTombstoned.has(rootId) ? normalizeTribe(entry, idx.tribes.get(rootId)) : null;
if (!tribe) { undo(); return null; }
if (matched.multi) {
const by = inviteTombstonedBy.get(matched.msgKey);
const revoked = !!by && (by.has(matched.inviteAuthor) || by.has(tribe.author));
const exposed = tribe.isAnonymous && clearOpenInvites.has(matched.msgKey);
if (revoked || exposed) { undo(); return null; }
}
return { matched, rootId, tribe, fresh, undo };
};
return {
type: 'tribe',
@ -359,6 +462,27 @@ module.exports = ({ cooler, tribeCrypto }) => {
return result;
},
async departures(tribeId) {
const idx = await buildTribeIndex();
const root = idx.rootByTip.get(tribeId) || tribeId;
const out = new Map();
const rootAuthor = idx.tribes.get(root) && idx.tribes.get(root).author;
let cur = root;
let prev = null;
let g = 0;
while (cur && g++ < 100000) {
const e = idx.tribes.get(cur);
if (!e) break;
const members = Array.isArray(e.content.members) ? e.content.members : [];
const noted = rootAuthor && e.author === rootAuthor && e.content.removedSeqs ? e.content.removedSeqs : {};
if (prev) for (const m of prev) if (!members.includes(m)) out.set(m, e.author === m ? { seq: e.seq || 0 } : (Number.isFinite(Number(noted[m])) && noted[m] !== null ? { seq: Number(noted[m]) } : { ts: e._ts || 0 }));
for (const m of members) out.delete(m);
prev = members;
cur = idx.child.get(cur);
}
return out;
},
async getRootId(tribeId) {
const idx = await buildTribeIndex();
let cur = tribeId;
@ -400,7 +524,7 @@ module.exports = ({ cooler, tribeCrypto }) => {
if (idx.tombstoned.has(latestId) || idx.effectivelyTombstoned.has(latestId)) throw new Error('Tribe not found');
const entry = idx.tribes.get(latestId);
if (!entry) throw new Error('Tribe not found');
return normalizeTribe(entry);
return normalizeTribe(entry, idx.tribes.get(idx.rootByTip.get(latestId) || latestId));
},
async listAll() {
@ -411,7 +535,7 @@ module.exports = ({ cooler, tribeCrypto }) => {
if (idx.effectivelyTombstoned.has(root) || idx.effectivelyTombstoned.has(tip)) continue;
const entry = idx.tribes.get(tip);
if (!entry) continue;
items.push(normalizeTribe(entry));
items.push(normalizeTribe(entry, idx.tribes.get(root)));
}
return items;
},
@ -452,6 +576,22 @@ module.exports = ({ cooler, tribeCrypto }) => {
return out;
},
async wasEverMember(userId, tribeId) {
if (!userId || !tribeId) return false;
const idx = await buildTribeIndex();
let root = tribeId;
while (idx.parent.has(root)) root = idx.parent.get(root);
for (const entry of idx.tribes.values()) {
let r = entry.id;
while (idx.parent.has(r)) r = idx.parent.get(r);
if (r !== root) continue;
if (entry.author === userId) return true;
const members = entry.content && Array.isArray(entry.content.members) ? entry.content.members : [];
if (members.includes(userId)) return true;
}
return false;
},
async isTribeMember(userId, tribeId) {
if (!userId || !tribeId) return false;
try {
@ -512,7 +652,9 @@ module.exports = ({ cooler, tribeCrypto }) => {
mapUrl: updatedContent.mapUrl !== undefined ? updatedContent.mapUrl : tribe.mapUrl,
createdAt: tribe.createdAt,
updatedAt: now,
author: tribe.author
author: tribe.author,
...(typeof updatedContent.joinedWith === 'string' && updatedContent.joinedWith ? { joinedWith: updatedContent.joinedWith } : {}),
...(updatedContent.removedSeqs && typeof updatedContent.removedSeqs === 'object' ? { removedSeqs: updatedContent.removedSeqs } : {})
};
if (fields.isAnonymous) {
return wrapAndPublish(rootId, { k: 'tribe', op: 'update', rootId, replaces: tipId, ...fields });
@ -533,7 +675,15 @@ module.exports = ({ cooler, tribeCrypto }) => {
async updateTribeMembers(tribeId, members) {
const tribe = await this.getTribeById(tribeId);
const old = tribe.members || [];
await this.updateTribeById(tribeId, { members });
const leaving = old.filter(m => !members.includes(m));
const removedSeqs = {};
if (leaving.length) {
const client = await openSsb();
for (const m of leaving) {
removedSeqs[m] = await new Promise((resolve) => pull(client.createUserStream({ id: m, reverse: true, limit: 1 }), pull.collect((err, list) => resolve(!err && list && list[0] && list[0].value ? Number(list[0].value.sequence) || 0 : 0))));
}
}
await this.updateTribeById(tribeId, { members, ...(leaving.length ? { removedSeqs } : {}) });
const removed = old.filter(m => !members.includes(m));
const added = members.filter(m => !old.includes(m));
if (removed.length > 0) {
@ -589,7 +739,7 @@ module.exports = ({ cooler, tribeCrypto }) => {
const root = await this.getRootId(tribeId).catch(() => tribeId);
const map = await scanOpenInviteMarkers();
const rec = map.get(root);
return rec ? { code: rec.code, by: rec.by, markerKey: rec.markerKey, inviteKey: rec.inviteKey } : null;
return rec ? { code: rec.code, by: rec.by, markerKey: rec.markerKey, inviteKey: rec.inviteKey, sealed: rec.sealed } : null;
},
async enrichOpenInvites(tribes) {
@ -627,9 +777,14 @@ module.exports = ({ cooler, tribeCrypto }) => {
const invitePub = await new Promise((resolve, reject) =>
client.publish({ type: 'tribe-invite-msg', v: 1, multi: 1, ch: codeHash, s: salt, ek: ekChain }, (err, r) => err ? reject(err) : resolve(r))
);
await new Promise((resolve, reject) =>
client.publish({ type: 'tribe-open-invite', v: 1, rootId: targetRoot, code, inviteKey: invitePub.key, by: userId, createdAt: new Date().toISOString() }, (err, r) => err ? reject(err) : resolve(r))
);
const marker = { v: 1, rootId: targetRoot, code, inviteKey: invitePub.key, by: userId, createdAt: new Date().toISOString() };
if (tribe.isAnonymous) {
await wrapAndPublish(targetRoot, { k: 'open-invite', ...marker });
} else {
await new Promise((resolve, reject) =>
client.publish({ type: 'tribe-open-invite', ...marker }, (err, r) => err ? reject(err) : resolve(r))
);
}
tribeIndex = null;
return code;
},
@ -643,56 +798,61 @@ module.exports = ({ cooler, tribeCrypto }) => {
if (rec.by !== userId && !(tribe && tribe.author === userId)) {
throw new Error('Not allowed to remove this invitation');
}
await new Promise((resolve, reject) =>
client.publish({ type: 'tribe-open-invite-tombstone', v: 1, target: rec.markerKey, ts: new Date().toISOString() }, (err, r) => err ? reject(err) : resolve(r))
);
const ts = new Date().toISOString();
if (rec.sealed) {
await wrapAndPublish(await this.getRootId(tribeId), { k: 'open-invite-tombstone', target: rec.markerKey, ts });
} else {
await new Promise((resolve, reject) =>
client.publish({ type: 'tribe-open-invite-tombstone', v: 1, target: rec.markerKey, ts }, (err, r) => err ? reject(err) : resolve(r))
);
}
if (rec.inviteKey) await this.publishInviteTombstone(rec.inviteKey).catch(() => {});
tribeIndex = null;
},
async previewInvite(rawCode) {
const found = await findInvite(String(rawCode || '').trim());
if (!found) return null;
found.undo();
return { rootId: found.rootId, title: found.tribe.title, tags: found.tribe.tags };
},
async joinByInvite(rawCode) {
const code = String(rawCode || '').trim();
if (!code) throw new Error('Invalid or expired invite code');
const client = await openSsb();
const userId = client.id;
const msgs = await streamLog();
const inviteTombstoned = new Set();
for (const m of msgs) {
const v = m.value;
const c = v && v.content;
if (!c || c.type !== 'tribe-invite-tombstone' || typeof c.target !== 'string') continue;
inviteTombstoned.add(c.target);
}
let matched = null;
for (const m of msgs) {
const v = m.value;
const c = v && v.content;
if (!c || c.type !== 'tribe-invite-msg' || c.v !== 1) continue;
if (typeof c.ch !== 'string' || typeof c.s !== 'string' || typeof c.ek !== 'string') continue;
if (inviteTombstoned.has(m.key)) continue;
if (tribeCrypto.hashInviteCode(code, c.s) !== c.ch) continue;
const chain = tribeCrypto.decryptChainFromInvite(c.ek, code, c.s, 3);
if (Array.isArray(chain) && chain.length) {
matched = { msgKey: m.key, codeHash: c.ch, chain, multi: c.multi === 1 || c.multi === true };
break;
}
}
if (!matched) throw new Error('Invalid or expired invite code');
for (const entry of matched.chain) {
tribeCrypto.setKeys(entry.rootId, entry.keys, entry.gen || entry.keys.length);
}
tribeIndex = null;
const rootId = matched.chain[0].rootId;
const found = await findInvite(code);
if (!found) throw new Error('Invalid or expired invite code');
const { matched, rootId, fresh, undo } = found;
let tribe;
try { tribe = await this.getTribeById(rootId); } catch (_) { tribe = null; }
if (!tribe) throw new Error('Tribe not found after key import');
if (tribe.members.includes(userId)) throw new Error('Already a member of this tribe');
if (!tribe) { undo(); throw new Error('Tribe not found'); }
if (tribe.members.includes(userId)) { undo(); throw new Error('Already a member of this tribe'); }
const lineage = new Set([rootId]);
let cursor = tribe;
for (let hops = 0; cursor && cursor.parentTribeId && hops < 8; hops++) {
let parentTribe = null;
try { parentTribe = await this.getTribeById(cursor.parentTribeId); } catch (_) { parentTribe = null; }
if (!parentTribe) break;
const parentCircle = [parentTribe.author, ...(Array.isArray(parentTribe.members) ? parentTribe.members : [])];
if (!parentCircle.includes(cursor.author)) break;
lineage.add(parentTribe.id);
lineage.add(cursor.parentTribeId);
cursor = parentTribe;
}
for (const entry of matched.chain) {
if (!entry || !Array.isArray(entry.keys) || !entry.keys.length) continue;
if (fresh.includes(entry)) { if (!lineage.has(entry.rootId)) tribeCrypto.dropKey(entry.rootId); continue; }
if (lineage.has(entry.rootId)) tribeCrypto.mergeKeys(entry.rootId, entry.keys, entry.gen || entry.keys.length);
}
tribeIndex = null;
const members = [...tribe.members, userId];
if (matched.multi) {
await this.updateTribeById(tribe.id, { members });
await this.updateTribeById(tribe.id, { members, joinedWith: matched.codeHash });
} else {
const invites = (tribe.invites || []).filter(inv => inv.codeHash !== matched.codeHash);
await this.updateTribeById(tribe.id, { members, invites });
await this.updateTribeById(tribe.id, { members, invites, joinedWith: matched.codeHash });
await this.publishInviteTombstone(matched.msgKey).catch(() => {});
}
await this.ensureFollowTribeMembers(tribe.id).catch(() => {});
@ -728,9 +888,6 @@ module.exports = ({ cooler, tribeCrypto }) => {
return;
}
await this.updateTribeById(tribeId, { members });
if (members.length > 0) {
await this.rotateTribeKey(tribeId, members).catch(() => {});
}
},
async distributeTribeKey(tribeId, toMembers) {
@ -741,13 +898,16 @@ module.exports = ({ cooler, tribeCrypto }) => {
const gen = tribeCrypto.getGen(rootId);
if (!keys.length) return;
const payload = tribeCrypto.buildKeyDistribPayload(rootId, keys, gen);
const batch = tribeCrypto.KEY_DISTRIB_BATCH;
for (let i = 0; i < toMembers.length; i += batch) {
const recps = toMembers.slice(i, i + batch);
const others = [...new Set(toMembers)].filter(m => m && m !== client.id);
const batch = tribeCrypto.KEY_DISTRIB_BATCH - 1;
let i = 0;
do {
const recps = [client.id, ...others.slice(i, i + batch)];
await new Promise((resolve, reject) =>
client.publish({ ...payload, recps }, (err) => err ? reject(err) : resolve())
);
}
i += batch;
} while (i < others.length);
},
async rotateTribeKey(tribeId, remainingMembers) {
@ -770,38 +930,41 @@ module.exports = ({ cooler, tribeCrypto }) => {
const rootId = await this.getRootId(tribeId);
if (!tribeCrypto.getKey(rootId)) return;
const others = (tribe.members || []).filter(m => m !== userId);
if (!others.length) return;
const ssbKeys = require('../server/node_modules/ssb-keys');
const config = require('../server/ssb_config');
const msgs = await streamLog();
const { distribs } = await readKeyDistribs();
const distributed = new Set();
for (const m of msgs) {
if (m.value && m.value.author !== userId) continue;
const c = m.value && m.value.content;
const dec = tribeCrypto.tryUnboxKeyDistrib(c, config.keys, ssbKeys);
if (!dec || dec.rootId !== rootId) continue;
const currentGen = tribeCrypto.getGen(rootId);
const holdersOfCurrent = new Set();
for (const { m, dec } of distribs) {
if (m.value.author !== userId) continue;
if (dec.rootId !== rootId) continue;
const recps = Array.isArray(dec.recps) ? dec.recps : [];
for (const r of recps) distributed.add(r);
if (Number(dec.gen) === Number(currentGen)) for (const r of recps) holdersOfCurrent.add(r);
}
const departed = [...holdersOfCurrent].filter(r => r !== userId && !(tribe.members || []).includes(r));
if (departed.length > 0) { await this.rotateTribeKey(tribeId, others).catch(() => {}); return; }
if (!others.length) return;
const missing = others.filter(m => !distributed.has(m));
if (missing.length > 0) await this.distributeTribeKey(tribeId, missing).catch(() => {});
},
async processIncomingKeys() {
const client = await openSsb();
const ssbKeys = require('../server/node_modules/ssb-keys');
const config = require('../server/ssb_config');
const msgs = await streamLog();
const { msgs, distribs } = await readKeyDistribs();
const byTribe = new Map();
for (const m of msgs) {
const c = m.value && m.value.content;
const dec = tribeCrypto.tryUnboxKeyDistrib(c, config.keys, ssbKeys);
if (!dec) continue;
const authorOf = new Map();
for (const m of msgs) if (m && m.key && m.value) authorOf.set(m.key, m.value.author);
for (const { m, dec } of distribs) {
const list = byTribe.get(dec.rootId) || [];
list.push({ generation: dec.gen || dec.keys.length, keys: dec.keys, ts: m.value.timestamp });
list.push({ generation: dec.gen || dec.keys.length, keys: dec.keys, ts: m.value.timestamp, author: m.value.author });
byTribe.set(dec.rootId, list);
}
for (const [rootId, entries] of byTribe.entries()) {
for (const [rootId, all] of byTribe.entries()) {
const owner = authorOf.get(rootId);
if (!owner) continue;
let members = [];
try { const t = await this.getTribeById(rootId); members = Array.isArray(t && t.members) ? t.members : []; } catch (_) {}
const allowed = new Set([owner, ...members]);
const entries = all.filter(e => allowed.has(e.author));
entries.sort((a, b) => b.generation - a.generation);
const top = entries[0];
if (top && Array.isArray(top.keys) && top.keys.length) {

285
src/models/typed_log.js Normal file
View file

@ -0,0 +1,285 @@
const pull = require('../server/node_modules/pull-stream');
const { AsyncLocalStorage } = require('async_hooks');
const requestScope = new AsyncLocalStorage();
const noteCapped = (limit, size) => {
if (!limit || !(size >= limit)) return;
const store = requestScope.getStore();
if (store) { store.capped = true; store.limit = limit; }
};
const TAIL_PROBE = 50;
const LOG_TAIL_PROBE = 20;
const collectStream = (stream) =>
new Promise((resolve, reject) => {
pull(stream, pull.collect((err, msgs) => (err ? reject(err) : resolve(msgs))));
});
const isDeletion = (m) => {
const c = m && m.value && m.value.content;
return !!c && typeof c === 'object' && c.type === 'tombstone';
};
const readContentWindow = (ssb, limit) => {
if (!limit) return collectStream(ssb.createLogStream({ reverse: true }));
let kept = 0;
return collectStream(pull(
ssb.createLogStream({ reverse: true }),
pull.take((m) => { if (!isDeletion(m)) kept++; return kept <= limit; })
));
};
const caches = new WeakMap();
const cacheFor = (ssb) => {
let c = caches.get(ssb);
if (!c) {
c = { types: new Map(), window: null, results: new Map() };
caches.set(ssb, c);
}
return c;
};
const insert = (entry, msgs) => {
for (let i = msgs.length - 1; i >= 0; i--) {
const m = msgs[i];
if (m && m.key && !entry.byKey.has(m.key)) entry.byKey.set(m.key, m);
}
};
const UNCAPPED_TYPES = new Set(['tombstone']);
const syncType = async (ssb, cache, type, requestedLimit) => {
const limit = UNCAPPED_TYPES.has(type) ? 0 : requestedLimit;
let entry = cache.types.get(type);
if (!entry) {
entry = { byKey: new Map(), warming: null, warm: false };
cache.types.set(type, entry);
entry.warming = collectStream(ssb.messagesByType(limit ? { type, reverse: true, limit } : { type, reverse: true }))
.then((msgs) => { insert(entry, msgs); entry.warm = true; entry.capped = !!(limit && msgs.length >= limit); })
.catch(() => { cache.types.delete(type); });
await entry.warming;
return entry;
}
await entry.warming;
if (!entry.warm) return entry;
const tail = await collectStream(ssb.messagesByType({ type, reverse: true, limit: TAIL_PROBE }));
const fresh = tail.filter((m) => m && m.key && !entry.byKey.has(m.key));
if (fresh.length === tail.length && tail.length === TAIL_PROBE) {
const all = await collectStream(ssb.messagesByType(limit ? { type, reverse: true, limit } : { type, reverse: true }));
insert(entry, all);
entry.capped = !!(limit && all.length >= limit);
} else {
insert(entry, fresh);
}
return entry;
};
const syncWindow = async (ssb, cache, limit) => {
let entry = cache.window;
if (!entry) {
entry = { byKey: new Map(), warming: null, warm: false };
cache.window = entry;
entry.warming = readContentWindow(ssb, limit)
.then((msgs) => { insert(entry, msgs); entry.warm = true; })
.catch(() => { cache.window = null; });
await entry.warming;
return entry;
}
await entry.warming;
if (!entry.warm) return entry;
const tail = await collectStream(ssb.createLogStream({ reverse: true, limit: TAIL_PROBE }));
const fresh = tail.filter((m) => m && m.key && !entry.byKey.has(m.key));
if (fresh.length === tail.length && tail.length === TAIL_PROBE) {
const all = await readContentWindow(ssb, limit);
insert(entry, all);
} else {
insert(entry, fresh);
}
return entry;
};
const syncPrivate = async (ssb, cache) => {
if (!ssb.private || typeof ssb.private.read !== 'function') return null;
let entry = cache.private;
if (!entry) {
entry = { byKey: new Map(), warming: null, warm: false };
cache.private = entry;
entry.warming = collectStream(ssb.private.read({ reverse: true }))
.then((msgs) => { insert(entry, msgs); entry.warm = true; })
.catch(() => { cache.private = null; });
await entry.warming;
return entry;
}
await entry.warming;
if (!entry.warm) return entry;
const tail = await collectStream(ssb.private.read({ reverse: true, limit: TAIL_PROBE }));
const fresh = tail.filter((m) => m && m.key && !entry.byKey.has(m.key));
if (fresh.length === tail.length && tail.length === TAIL_PROBE) {
insert(entry, await collectStream(ssb.private.read({ reverse: true })));
} else {
insert(entry, fresh);
}
return entry;
};
const logTailOf = async (ssb, n) => {
if (ssb.activity && typeof ssb.activity.tail === 'function') {
const store = requestScope.getStore();
const fresh = !(store && store.method === 'GET');
const fast = await new Promise((resolve) => {
try { ssb.activity.tail(n, { fresh }, (err, arr) => resolve(err ? null : arr)); } catch (_) { resolve(null); }
});
if (Array.isArray(fast)) return fast;
}
return collectStream(ssb.createLogStream({ reverse: true, limit: n }));
};
const readTyped = async (ssbClient, types, opts = {}) => {
const limit = opts.limit;
if (typeof ssbClient.messagesByType !== 'function') {
const all = await collectStream(ssbClient.createLogStream(limit ? { limit } : {}));
noteCapped(limit, all.length);
return all;
}
const cache = cacheFor(ssbClient);
const wanted = new Set(types);
const logTail = (await logTailOf(ssbClient, LOG_TAIL_PROBE)).reverse();
const tipKey = logTail.length ? logTail[logTail.length - 1].key : null;
const inTail = new Set(logTail.map((m) => m && m.key));
const upToDate = (e) => !!e && e.warm && e.tip === tipKey;
const caughtUp = (e) => !!e && e.warm && !!e.tip && (e.tip === tipKey || inTail.has(e.tip));
const allWarm = types.every((type) => upToDate(cache.types.get(type)))
&& (!opts.withWindow || upToDate(cache.window));
const entries = await Promise.all(types.map((type) => {
const e = cache.types.get(type);
return caughtUp(e) ? e : syncType(ssbClient, cache, type, limit);
}));
const windowEntry = opts.withWindow
? (caughtUp(cache.window) ? cache.window : await syncWindow(ssbClient, cache, limit))
: null;
const withPrivate = opts.withPrivate === true;
const resultKey = `${types.join(',')}|${limit || 0}|${opts.withWindow ? 1 : 0}|${withPrivate ? 1 : 0}`;
if (allWarm && (!withPrivate || !wanted.size || upToDate(cache.private))) {
const hit = cache.results.get(resultKey);
if (hit && hit.tip === tipKey) {
if (hit.capped) noteCapped(limit, limit);
return hit.list.slice();
}
}
const privateEntry = withPrivate && wanted.size
? (!upToDate(cache.private) ? await syncPrivate(ssbClient, cache) : cache.private)
: null;
for (const e of [...entries, windowEntry, privateEntry]) if (e && e.warm) e.tip = tipKey;
for (const m of logTail) {
if (!m || !m.key || !m.value) continue;
const t = m.value.content && m.value.content.type;
if (typeof t === 'string' && wanted.has(t)) {
const entry = cache.types.get(t);
if (entry && entry.warm && !entry.byKey.has(m.key)) entry.byKey.set(m.key, m);
}
if (windowEntry && windowEntry.warm && !windowEntry.byKey.has(m.key)) windowEntry.byKey.set(m.key, m);
}
const union = new Map();
let capped = false;
for (const entry of entries) {
if (!entry) continue;
if (entry.capped) { capped = true; noteCapped(limit, limit); }
for (const [k, m] of entry.byKey) if (!union.has(k)) union.set(k, m);
}
if (windowEntry) {
for (const [k, m] of windowEntry.byKey) if (!union.has(k)) union.set(k, m);
}
if (privateEntry && privateEntry.warm) {
for (const [k, m] of privateEntry.byKey) {
const t = m && m.value && m.value.content && m.value.content.type;
if (typeof t === 'string' && wanted.has(t)) union.set(k, m);
}
}
const sorted = Array.from(union.values()).sort((a, b) => {
const at = (a.value && a.value.timestamp) || 0;
const bt = (b.value && b.value.timestamp) || 0;
if (at !== bt) return at - bt;
const ar = a.timestamp || 0;
const br = b.timestamp || 0;
if (ar !== br) return ar - br;
if (a.value && b.value && a.value.author === b.value.author) return (a.value.sequence || 0) - (b.value.sequence || 0);
return 0;
});
const complete = entries.every((e) => e && e.warm)
&& (!windowEntry === !opts.withWindow) && (!windowEntry || windowEntry.warm)
&& (!(withPrivate && wanted.size) || (privateEntry && privateEntry.warm));
if (tipKey && complete) cache.results.set(resultKey, { tip: tipKey, capped, list: sorted });
return sorted.slice();
};
const NON_MESSAGE_LITERALS = new Set([
'hidden', 'submit', 'text', 'file', 'number', 'checkbox', 'radio', 'password', 'date', 'datetime-local', 'time', 'url', 'email', 'search', 'button', 'reset',
'png', 'jpeg', 'jpg', 'gif', 'svg', 'webp', 'pdf', 'zip', 'json', 'html', 'error', 'meta', 'msg', 'blob', 'peer', 'inhabitant', 'chatThread', 'taskAssignment'
]);
const CORE_TYPES = ['post', 'about', 'contact', 'vote', 'pub', 'tombstone', 'file', 'textChunk'];
const discoverContentTypes = () => {
const fs = require('fs');
const path = require('path');
const found = new Set(CORE_TYPES);
const sources = [];
try {
for (const f of fs.readdirSync(__dirname)) if (f.endsWith('.js') && f !== 'typed_log.js') sources.push(path.join(__dirname, f));
sources.push(path.join(__dirname, '..', 'backend', 'backend.js'));
} catch (_) {}
for (const file of sources) {
let src = '';
try { src = fs.readFileSync(file, 'utf8'); } catch (_) { continue; }
for (const m of src.matchAll(/\btype:\s*['"]([A-Za-z][\w-]*)['"]/g)) {
if (!NON_MESSAGE_LITERALS.has(m[1])) found.add(m[1]);
}
for (const m of src.matchAll(/\b(?:const|let)\s+(?:[A-Z][A-Z0-9_]*_)?TYPE\s*=\s*['"]([A-Za-z][\w-]*)['"]/g)) {
if (!NON_MESSAGE_LITERALS.has(m[1])) found.add(m[1]);
}
}
return Array.from(found).sort();
};
const CONTENT_TYPES = discoverContentTypes();
const logTip = async (ssb) => {
const tail = await logTailOf(ssb, 1);
return tail[0] && tail[0].key ? tail[0].key : null;
};
const activityCache = new WeakMap();
const ACTIVITY_TTL_MS = 5000;
const authorActivity = (ssb) => {
if (!ssb || !ssb.activity || typeof ssb.activity.latest !== 'function') return Promise.resolve(null);
const hit = activityCache.get(ssb);
if (hit && Date.now() - hit.at < ACTIVITY_TTL_MS) return hit.pending;
const store = requestScope.getStore();
const fresh = !(store && store.method === 'GET');
const pending = new Promise((resolve) => {
try { ssb.activity.latest(null, { fresh }, (err, all) => resolve(err || !all ? null : new Map(Object.entries(all)))); } catch (_) { resolve(null); }
});
activityCache.set(ssb, { at: Date.now(), pending });
return pending;
};
const memoIndex = (name, messages, build) => {
const store = requestScope.getStore();
if (!store) return build(messages);
const list = Array.isArray(messages) ? messages : [];
const first = list[0];
const last = list[list.length - 1];
const sig = `${list.length}|${first && first.key}|${last && last.key}`;
if (!store.indexMemo) store.indexMemo = new Map();
const hit = store.indexMemo.get(name);
if (hit && hit.sig === sig) return hit.idx;
const idx = build(messages);
store.indexMemo.set(name, { sig, idx });
return idx;
};
module.exports = { readTyped, collectStream, readContentWindow, CONTENT_TYPES, discoverContentTypes, requestScope, memoIndex, logTip, authorActivity };

78
src/models/videos_model.js
View file

@ -1,7 +1,8 @@
const pull = require("../server/node_modules/pull-stream");
const { getConfig } = require("../configs/config-manager.js");
const { buildValidatedTombstoneSet } = require('./tombstone_validator');
const { dedupeBy, norm } = require('../backend/dedupe');
const { dedupeBy, norm, versionGroups } = require('../backend/dedupe');
const { readTyped, memoIndex } = require('./typed_log');
const categories = require("../backend/opinion_categories");
const logLimit = getConfig().ssbLogStream?.limit || 1000;
@ -31,13 +32,9 @@ module.exports = ({ cooler }) => {
return ssb;
};
const getAllMessages = async (ssbClient) =>
new Promise((resolve, reject) => {
pull(
ssbClient.createLogStream({ limit: logLimit }),
pull.collect((err, msgs) => (err ? reject(err) : resolve(msgs)))
);
});
const VIDEO_TYPES = ["video", "videoOpinion", "tombstone"];
const getAllMessages = async (ssbClient) => readTyped(ssbClient, VIDEO_TYPES, { limit: logLimit });
const getMsg = async (ssbClient, key) =>
new Promise((resolve, reject) => {
@ -47,9 +44,6 @@ module.exports = ({ cooler }) => {
const buildIndex = (messages) => {
const tomb = buildValidatedTombstoneSet(messages);
const nodes = new Map();
const parent = new Map();
const child = new Map();
const strictChild = new Map();
const opinionMsgs = [];
for (const m of messages) {
@ -65,41 +59,16 @@ module.exports = ({ cooler }) => {
const ts = v.timestamp || m.timestamp || 0;
let sizeBytes = 0;
try { sizeBytes = Buffer.byteLength(JSON.stringify(v), "utf8"); } catch (_) { sizeBytes = 0; }
nodes.set(k, { key: k, ts, c, sizeBytes, author: v.author });
if (c.replaces) {
parent.set(k, c.replaces);
child.set(c.replaces, k);
}
nodes.set(k, { key: k, ts, seq: v.sequence || 0, c, sizeBytes, author: v.author });
}
for (const [k, node] of nodes) {
const t = node.c.replaces;
if (t) { const orig = nodes.get(t); if (orig && orig.author === node.author) strictChild.set(t, k); }
}
const rootOf = (id) => {
let cur = id, g = 0;
while (parent.has(cur) && nodes.has(parent.get(cur)) && g++ < 100000) cur = parent.get(cur);
return cur;
};
const contentTipOf = (root) => {
let cur = root, g = 0;
while (strictChild.has(cur) && g++ < 100000) cur = strictChild.get(cur);
const n = nodes.get(cur), rn = nodes.get(root);
return (n && rn && n.author === rn.author) ? cur : root;
};
const roots = new Set();
for (const id of nodes.keys()) roots.add(rootOf(id));
const vg = versionGroups([...nodes.values()].map(n => ({ key: n.key, author: n.author, seq: n.seq, ts: n.ts, replaces: n.c.replaces || null, claimedAuthor: typeof n.c.author === "string" ? n.c.author : null })));
const rootOf = vg.rootOf;
const contentTipOf = vg.tipOf;
const opinionsByRoot = new Map();
for (const op of opinionMsgs) { if (!nodes.has(op.target)) continue; const r = rootOf(op.target); if (!opinionsByRoot.has(r)) opinionsByRoot.set(r, []); opinionsByRoot.get(r).push(op); }
const forward = new Map();
for (const [newId, oldId] of parent.entries()) forward.set(oldId, newId);
const resolveGroup = (root) => {
const contentTip = contentTipOf(root);
const contentNode = nodes.get(contentTip) || nodes.get(root);
@ -116,9 +85,9 @@ module.exports = ({ cooler }) => {
};
const tipByRoot = new Map();
for (const r of roots) tipByRoot.set(r, contentTipOf(r));
for (const r of vg.roots) tipByRoot.set(r, contentTipOf(r));
return { tomb, nodes, parent, child, rootOf, tipByRoot, forward, resolveGroup };
return { tomb, nodes, rootOf, tipByRoot, resolveGroup };
};
const buildVideo = (node, rootId, viewerId, agg) => {
@ -132,9 +101,10 @@ module.exports = ({ cooler }) => {
createdAt: c.createdAt || new Date(node.ts).toISOString(),
updatedAt: c.updatedAt || null,
tags: safeArr(c.tags),
author: c.author,
author: node.author,
title: c.title || "",
description: c.description || "",
license: c.license || "",
mapUrl: c.mapUrl || "",
opinions,
opinions_inhabitants: voters,
@ -161,16 +131,12 @@ module.exports = ({ cooler }) => {
const messages = await getAllMessages(ssbClient);
const idx = buildIndex(messages);
let tip = id;
while (idx.forward.has(tip)) tip = idx.forward.get(tip);
if (idx.tomb.has(tip)) throw new Error("Video not found");
let root = tip;
while (idx.parent.has(root)) root = idx.parent.get(root);
const root = idx.rootOf(id);
if (idx.tomb.has(idx.tipByRoot.get(root) || id)) throw new Error("Video not found");
return root;
},
async createVideo(blobMarkdown, tagsRaw, title, description, mapUrl) {
async createVideo(blobMarkdown, tagsRaw, title, description, mapUrl, license) {
const ssbClient = await openSsb();
const blobId = parseBlobId(blobMarkdown);
const tags = normalizeTags(tagsRaw) || [];
@ -187,7 +153,8 @@ module.exports = ({ cooler }) => {
description: description || "",
mapUrl: mapUrl || "",
opinions: {},
opinions_inhabitants: []
opinions_inhabitants: [],
...(license ? { license } : {})
};
return new Promise((resolve, reject) => {
@ -195,7 +162,7 @@ module.exports = ({ cooler }) => {
});
},
async updateVideoById(id, blobMarkdown, tagsRaw, title, description, mapUrl) {
async updateVideoById(id, blobMarkdown, tagsRaw, title, description, mapUrl, license) {
const ssbClient = await openSsb();
const userId = ssbClient.id;
const tipId = await this.resolveCurrentId(id);
@ -219,7 +186,8 @@ module.exports = ({ cooler }) => {
description: description !== undefined ? description || "" : oldMsg.content.description || "",
mapUrl: mapUrl !== undefined ? mapUrl || "" : oldMsg.content.mapUrl || "",
createdAt: oldMsg.content.createdAt,
updatedAt: now
updatedAt: now,
...(license !== undefined ? { license } : {})
};
const tombstone = { type: "tombstone", target: tipId, deletedAt: now, author: userId };
@ -304,10 +272,10 @@ module.exports = ({ cooler }) => {
const ssbClient = await openSsb();
const viewer = viewerId || ssbClient.id;
const messages = await getAllMessages(ssbClient);
const idx = buildIndex(messages);
const idx = memoIndex('videos', messages, buildIndex);
const rootId = idx.rootOf(id);
const agg = idx.resolveGroup(rootId);
if (!agg.contentNode || idx.tomb.has(agg.contentTip)) throw new Error("Video not found");
if (!idx.tipByRoot.has(rootId) || !agg.contentNode || idx.tomb.has(agg.contentTip)) throw new Error("Video not found");
return buildVideo(agg.contentNode, rootId, viewer, agg);
},

88
src/models/viewer_filters.js
View file

@ -3,7 +3,7 @@ const path = require('path');
const { getConfig } = require('../configs/config-manager.js');
const COOLDOWN_MS = 5 * 60 * 1000;
const statePath = path.join(__dirname, '../configs/follow_state.json');
const statePath = require('../configs/state-manager').statePath('follow_state.json');
const readJson = (p, fallback) => {
try { return JSON.parse(fs.readFileSync(p, 'utf8')); } catch (e) { return fallback; }
@ -21,6 +21,87 @@ const loadState = () => {
};
const saveState = (s) => writeJson(statePath, s);
const lanPeersPath = require('../configs/state-manager').statePath('lan-peers.json');
const FEED_ID = /^@[A-Za-z0-9+/]{43}=\.ed25519$/;
let lanKnown = null;
const canonicalFeedId = (key) => {
let k = String(key || '').trim();
if (!k) return null;
if (!k.startsWith('@')) k = '@' + k;
if (!k.endsWith('.ed25519')) k += '.ed25519';
return FEED_ID.test(k) ? k : null;
};
const isPrivateHost = (host) => {
const h = String(host || '').replace(/^\[|\]$/g, '').replace(/^::ffff:/i, '').toLowerCase();
if (!h) return false;
if (/^(10|127)\./.test(h) || /^192\.168\./.test(h) || /^169\.254\./.test(h)) return true;
const m = h.match(/^172\.(\d+)\./);
if (m && Number(m[1]) >= 16 && Number(m[1]) <= 31) return true;
return h === '::1' || /^f[cd][0-9a-f]{0,2}:/.test(h) || /^fe[89ab][0-9a-f]?:/.test(h);
};
const hostOfAddress = (address) => {
const net = String(address || '').split('~')[0];
if (!net.startsWith('net:')) return '';
const hp = net.slice(4);
const i = hp.lastIndexOf(':');
return i > 0 ? hp.slice(0, i) : hp;
};
const lanKeysFromConn = (entries) => {
const out = [];
for (const e of entries || []) {
if (!Array.isArray(e) || !e[1]) continue;
const [address, data] = e;
const key = canonicalFeedId(data.key);
if (!key) continue;
if (data.type === 'lan' || data.inferredType === 'lan' || isPrivateHost(hostOfAddress(address))) out.push(key);
}
return out;
};
const lanPeers = () => {
if (!lanKnown) {
const list = readJson(lanPeersPath, []);
lanKnown = new Set((Array.isArray(list) ? list : []).map(canonicalFeedId).filter(Boolean));
}
return lanKnown;
};
const rememberLanPeers = (keys) => {
const set = lanPeers();
let changed = false;
for (const k of keys || []) {
const id = canonicalFeedId(k);
if (id && !set.has(id)) { set.add(id); changed = true; }
}
if (changed) writeJson(lanPeersPath, [...set]);
return changed;
};
const MUTUAL_CONCURRENCY = 16;
const filterByWish = async (items, { wish, viewer, authorOf: author, isOwn, isMutual, lan } = {}) => {
if (!Array.isArray(items)) return items;
const own = (it) => !!(isOwn && isOwn(it, viewer));
if (wish === 'local') return items.filter(own);
if (wish === 'only-lan') {
const near = lan || new Set();
return items.filter(it => { const a = author(it); return !a || a === viewer || own(it) || near.has(a); });
}
if (wish === 'mutuals' && typeof isMutual === 'function') {
const authors = [...new Set(items.map(author).filter(a => a && a !== viewer))];
const verdict = new Map();
let next = 0;
await Promise.all(Array.from({ length: Math.min(MUTUAL_CONCURRENCY, authors.length) }, async () => {
while (next < authors.length) { const a = authors[next++]; verdict.set(a, !!(await isMutual(a))); }
}));
return items.filter(it => { const a = author(it); return !a || a === viewer || verdict.get(a); });
}
return items;
};
const wishMutualsOnly = () => getConfig().wish === 'mutuals';
const pmMutualsOnly = () => getConfig().pmVisibility === 'mutuals';
const isFrictionActive = () => wishMutualsOnly() || pmMutualsOnly();
@ -127,4 +208,9 @@ module.exports = {
applyMutualSupportFilter,
canSendPmTo,
authorOf,
isPrivateHost,
lanKeysFromConn,
lanPeers,
rememberLanPeers,
filterByWish,
};

41
src/models/votes_model.js
View file

@ -5,9 +5,18 @@ const { getConfig } = require('../configs/config-manager.js');
const { dedupeBy, norm } = require('../backend/dedupe');
const { buildVoteTally } = require('../backend/vote_tally');
const categories = require('../backend/opinion_categories');
const { readTyped, memoIndex } = require('./typed_log');
const logLimit = getConfig().ssbLogStream?.limit || 1000;
const MIN_VOTE_DAYS = 7;
const INSTITUTIONAL_VOTE_TAG = /^(gov|govMethod|courtsCase|courtsMethod):/;
const isVoteLocked = (v) => !!v && (
(parseInt(v.totalVotes || 0, 10) || 0) > 0
|| Object.values(v.opinions || {}).some(n => (Number(n) || 0) > 0)
|| (Array.isArray(v.tags) && v.tags.some(t => INSTITUTIONAL_VOTE_TAG.test(String(t))))
);
const lockedError = () => Object.assign(new Error('Vote locked'), { code: 'VOTE_LOCKED' });
module.exports = ({ cooler }) => {
let ssb;
const openSsb = async () => {
@ -17,13 +26,10 @@ module.exports = ({ cooler }) => {
const TYPE = 'votes';
const VOTES_TYPES = [TYPE, 'votesVote', 'votesOpinion', 'tombstone'];
async function getAllMessages(ssbClient) {
return new Promise((resolve, reject) => {
pull(
ssbClient.createLogStream({ limit: logLimit }),
pull.collect((err, results) => (err ? reject(err) : resolve(results)))
);
});
return readTyped(ssbClient, VOTES_TYPES, { limit: logLimit });
}
function buildIndex(messages) {
@ -220,6 +226,7 @@ module.exports = ({ cooler }) => {
ssbClient.get(tipId, (err, msg) => (err || !msg ? rej(new Error('Vote not found')) : res(msg)))
);
if (!vote.content || vote.content.createdBy !== userId) throw new Error('Not the author');
if (isVoteLocked(vote.content) || isVoteLocked(await this.getVoteById(id).catch(() => null))) throw lockedError();
const tombstone = {
type: 'tombstone',
target: tipId,
@ -246,7 +253,7 @@ module.exports = ({ cooler }) => {
if (c.createdBy !== userId) throw new Error('Not the author');
const agg = await this.getVoteById(id);
const aggTotalVotes = agg ? (parseInt(agg.totalVotes || 0, 10) || 0) : 0;
if (agg && Object.keys(agg.opinions || {}).some(k => (agg.opinions[k] || 0) > 0)) throw new Error('Cannot edit vote after it has received opinions.');
if (isVoteLocked(c) || isVoteLocked(agg)) throw lockedError();
let newDeadline = c.deadline;
if (deadline != null && deadline !== '') {
@ -324,7 +331,7 @@ module.exports = ({ cooler }) => {
async getVoteById(id) {
const ssbClient = await openSsb();
const messages = await getAllMessages(ssbClient);
const index = buildIndex(messages);
const index = memoIndex('votes', messages, buildIndex);
const activeList = computeActiveVotes(index);
const byId = new Map(activeList.map(v => [v.id, v]));
@ -339,19 +346,7 @@ module.exports = ({ cooler }) => {
return candidate;
}
const msg = await new Promise((res, rej) =>
ssbClient.get(id, (err, vote) => (err || !vote ? rej(new Error('Vote not found')) : res(vote)))
);
const content = msg.content || {};
const status = statusFromContent(content, moment());
return {
id,
latestId: id,
...content,
status
};
throw new Error('Vote not found');
},
async listAll(filter = 'all') {
@ -363,6 +358,8 @@ module.exports = ({ cooler }) => {
if (filter === 'mine') {
list = list.filter(v => v.createdBy === userId);
} else if (filter === 'recent') {
list = list.filter(v => (Date.parse(v.createdAt || '') || 0) >= Date.now() - 86400000);
} else if (filter === 'open') {
list = list.filter(v => v.status === 'OPEN');
} else if (filter === 'closed') {
@ -389,4 +386,4 @@ module.exports = ({ cooler }) => {
}
};
};
module.exports.isVoteLocked = isVoteLocked;

23
src/models/wallet_model.js
View file

@ -1,16 +1,19 @@
const {
RequestManager,
HTTPTransport,
Client
} = require("../server/node_modules/@open-rpc/client-js");
const RPC_TIMEOUT_MS = 30000;
async function makeClient(url, user, pass) {
const headers = {};
const headers = { 'Content-Type': 'application/json' };
if (user !== undefined || pass !== undefined) {
headers['Authorization'] = 'Basic ' + Buffer.from(`${user || ''}:${pass || ''}`).toString('base64');
}
const transport = new HTTPTransport(url, { headers });
return new Client(new RequestManager([transport]));
let nextId = 0;
return {
request: async ({ method, params = [] }) => {
const res = await fetch(url, { method: 'POST', headers, body: JSON.stringify({ jsonrpc: '1.0', id: ++nextId, method, params }), signal: AbortSignal.timeout(RPC_TIMEOUT_MS) });
const body = await res.json();
if (!body || body.error) throw new Error((body && body.error && body.error.message) || `RPC ${res.status}`);
return body.result;
}
};
}
module.exports = {
@ -50,8 +53,8 @@ module.exports = {
const errors = [];
const addrInfo = await module.exports.execute(url, user, pass, "validateaddress", [address]);
const addressValid = !!addrInfo?.isvalid;
const amountValid = Number(amount) > 0;
const feeValid = Number(fee) >= 0;
const amountValid = Number.isFinite(Number(amount)) && Number(amount) > 0;
const feeValid = Number.isFinite(Number(fee)) && Number(fee) >= 0;
if (!addressValid) errors.push("invalid_dest");
if (!amountValid) errors.push("invalid_amount");
if (!feeValid) errors.push("invalid_fee");

485
src/models/wiki_model.js Normal file
View file

@ -0,0 +1,485 @@
const pull = require('../server/node_modules/pull-stream');
const { readTyped, memoIndex } = require('./typed_log');
const { buildValidatedTombstoneSet } = require('./tombstone_validator');
const { getConfig } = require('../configs/config-manager.js');
const logLimit = getConfig().ssbLogStream?.limit || 1000;
const WIKILINK_RE = /\[\[([^\]|\n]{1,120})(?:\|([^\]\n]{1,120}))?\]\]/g;
const slugify = (value) => String(value == null ? '' : value)
.normalize('NFD')
.replace(/[̀-ͯ]/g, '')
.toLowerCase()
.trim()
.replace(/[^a-z0-9]+/g, '-')
.replace(/^-+|-+$/g, '')
.slice(0, 80);
const linkTarget = (raw) => {
const s = String(raw || '').trim();
const isUrl = /^[a-z]+:\/\//i.test(s);
const wikiPath = (isUrl || s.startsWith('/')) ? s.match(/\/wiki\/([^?#]+)/) : null;
if (wikiPath) { try { return decodeURIComponent(wikiPath[1]).trim(); } catch (_) { return wikiPath[1].trim(); } }
if (isUrl) {
const seg = s.replace(/[?#].*$/, '').split('/').filter(Boolean).pop() || '';
try { return decodeURIComponent(seg).trim(); } catch (_) { return seg.trim(); }
}
const namespaced = s.match(/^[a-z]+:(?=\S)(.+)$/i);
return namespaced ? namespaced[1].trim() : s;
};
const extractWikiLinks = (text) => {
const out = [];
const seen = new Set();
for (const m of String(text || '').matchAll(WIKILINK_RE)) {
const slug = slugify(linkTarget(m[1]));
if (!slug || seen.has(slug)) continue;
seen.add(slug);
out.push(slug);
}
return out;
};
const WIKI_TYPE = 'wikiPage';
const ENVELOPE_TYPE = 'tribe-msg';
const EDIT_POLICIES = ['open', 'author', 'tribe'];
const RECENT_MS = 24 * 60 * 60 * 1000;
const MAX_CHANGES = 60;
const MAX_TITLE = 100;
const IMAGE_RE = /!\[image:[^\]\[\n]{0,160}\]\((&[^)\s]{1,120}\.sha256)\)/;
const BODY_INLINE_MAX = 6000;
const bodyCache = new Map();
const addBlob = (ssbClient, buf) => new Promise((resolve, reject) => pull(pull.values([buf]), ssbClient.blobs.add((err, ref) => err ? reject(err) : resolve(ref))));
const getBlob = (ssbClient, ref) => new Promise((resolve) => {
try {
pull(ssbClient.blobs.get(ref), pull.collect((err, chunks) => resolve(err || !chunks || !chunks.length ? null : Buffer.concat(chunks.map(c => Buffer.isBuffer(c) ? c : Buffer.from(c))))));
} catch (_) { resolve(null); }
});
const safeText = (v) => String(v == null ? '' : v).trim();
const normalizeList = (raw) => {
if (!raw) return [];
const arr = Array.isArray(raw) ? raw : String(raw).split(',');
return Array.from(new Set(arr.map(t => String(t || '').trim()).filter(Boolean)));
};
const normalizePolicy = (v) => {
const s = String(v || '').toLowerCase();
if (s === 'closed') return 'author';
return EDIT_POLICIES.includes(s) ? s : 'open';
};
module.exports = ({ cooler, tribeCrypto = null, tribesModel = null }) => {
let ssb;
const openSsb = async () => { if (!ssb) ssb = await cooler.open(); return ssb; };
const tribeRootId = async (tribeId) => {
if (!tribesModel || !tribeId) return tribeId;
try { return await tribesModel.getRootId(tribeId); } catch (_) { return tribeId; }
};
const tribeKeyFor = async (tribeId) => {
if (!tribeCrypto || !tribeId) return null;
const rootId = await tribeRootId(tribeId);
return tribeCrypto.getKey(rootId) || null;
};
const storeBody = async (ssbClient, body, tribeId) => {
const text = safeText(body);
if (Buffer.byteLength(text, 'utf8') <= BODY_INLINE_MAX) return { body: text };
let payload = text;
if (tribeId) {
const key = await tribeKeyFor(tribeId);
if (!key) throw new Error('Missing tribe key — cannot publish encrypted wiki page');
payload = JSON.stringify(tribeCrypto.encryptContent({ body: text }, [key], true));
}
const ref = await addBlob(ssbClient, Buffer.from(payload, 'utf8'));
bodyCache.set(ref, text);
return { body: '', bodyBlob: ref };
};
const resolveBody = async (ssbClient, c) => {
if (!c || typeof c.bodyBlob !== 'string' || !c.bodyBlob) return c;
if (bodyCache.has(c.bodyBlob)) return { ...c, body: bodyCache.get(c.bodyBlob) };
const buf = await getBlob(ssbClient, c.bodyBlob);
if (!buf) return { ...c, body: safeText(c.body) };
let text = buf.toString('utf8');
if (c.tribeId) {
try {
const key = await tribeKeyFor(c.tribeId);
const r = tribeCrypto.decryptContent(JSON.parse(text), key ? [[key]] : []);
text = r && !r._undecryptable && typeof r.body === 'string' ? r.body : '';
} catch (_) { text = ''; }
}
bodyCache.set(c.bodyBlob, text);
return { ...c, body: text };
};
const readEntries = async (ssbClient) => {
const msgs = await readTyped(ssbClient, [WIKI_TYPE, 'tombstone', ENVELOPE_TYPE], { limit: logLimit });
const fpIdx = tribeCrypto && typeof tribeCrypto.buildFingerprintIndex === 'function' ? tribeCrypto.buildFingerprintIndex() : null;
const out = [];
for (const m of msgs) {
const v = m && m.value;
const c = v && v.content;
if (!c || typeof c !== 'object') continue;
if (c.type === WIKI_TYPE || c.type === 'tombstone') { out.push(m); continue; }
if (!fpIdx || !tribeCrypto.isTribeMsg(c)) continue;
const r = tribeCrypto.unwrapMsg(c, fpIdx);
if (!r || !r.body || typeof r.body !== 'object') continue;
const body = r.body;
if (body.k === WIKI_TYPE) {
const flat = { ...body, type: WIKI_TYPE };
delete flat.k;
out.push({ ...m, value: { ...v, content: flat } });
} else if (body.k === 'tombstone' && typeof body.target === 'string') {
out.push({ ...m, value: { ...v, content: { type: 'tombstone', target: body.target, deletedAt: body.deletedAt, author: body.author || v.author } } });
}
}
for (let i = 0; i < out.length; i++) {
const c = out[i].value && out[i].value.content;
if (c && c.type === WIKI_TYPE && c.bodyBlob) out[i] = { ...out[i], value: { ...out[i].value, content: await resolveBody(ssbClient, c) } };
}
return out;
};
const tribeMembersFor = async (entries) => {
const out = new Map();
if (!tribesModel) return out;
for (const m of entries) {
const c = m.value.content;
if (c.type !== WIKI_TYPE || !c.tribeId || out.has(String(c.tribeId))) continue;
let members = [];
try { const t = await tribesModel.getTribeById(c.tribeId); members = Array.isArray(t && t.members) ? t.members : []; } catch (_) {}
out.set(String(c.tribeId), members);
}
return out;
};
const buildIndex = (entries, membersOf = new Map()) => {
const tomb = buildValidatedTombstoneSet(entries);
const nodes = new Map();
for (const m of entries) {
const c = m.value.content;
if (c.type !== WIKI_TYPE) continue;
nodes.set(m.key, { key: m.key, author: m.value.author, ts: m.value.timestamp || m.timestamp || 0, c });
}
const mayEdit = (state, author) => {
if (author === state.owner) return true;
if (state.policy === 'author') return false;
if (state.tribeId) return (membersOf.get(state.tribeId) || []).includes(author);
return state.policy !== 'tribe';
};
const states = new Map();
const stateOf = (key) => {
const path = [];
const seen = new Set();
let cur = key;
while (!states.has(cur)) {
const node = nodes.get(cur);
if (!node) { states.set(cur, null); break; }
if (typeof node.c.replaces !== 'string') {
states.set(cur, { root: cur, owner: node.author, tribeId: String(node.c.tribeId || ''), policy: normalizePolicy(node.c.editPolicy), license: safeText(node.c.license) });
break;
}
if (seen.has(cur)) { states.set(cur, null); break; }
seen.add(cur);
path.push(cur);
cur = node.c.replaces;
}
for (let i = path.length - 1; i >= 0; i--) {
const k = path[i];
if (states.has(k)) continue;
const node = nodes.get(k);
const parent = states.get(node.c.replaces);
if (!parent || String(node.c.tribeId || '') !== parent.tribeId || !mayEdit(parent, node.author)) { states.set(k, null); continue; }
states.set(k, node.author === parent.owner ? { ...parent, policy: normalizePolicy(node.c.editPolicy), license: safeText(node.c.license) } : parent);
}
return states.get(key);
};
const rootOf = (key) => { const s = stateOf(key); return s ? s.root : key; };
const versionsByRoot = new Map();
for (const node of nodes.values()) {
if (!stateOf(node.key)) continue;
const root = rootOf(node.key);
if (tomb.has(root)) continue;
if (!versionsByRoot.has(root)) versionsByRoot.set(root, []);
versionsByRoot.get(root).push(node);
}
const pages = [];
for (const [root, versions] of versionsByRoot) {
versions.sort((a, b) => a.ts - b.ts);
const first = versions[0];
const tip = versions[versions.length - 1];
const state = stateOf(tip.key);
const c = tip.c;
const body = safeText(c.body);
pages.push({
id: root,
rootId: root,
tipId: tip.key,
slug: slugify(c.title) || safeText(c.slug),
title: safeText(c.title).slice(0, MAX_TITLE),
body,
image: (body.match(IMAGE_RE) || [])[1] || null,
tags: normalizeList(c.tags),
aliases: [...new Set([...normalizeList(c.aliases).map(slugify), safeText(c.slug)].filter(a => a && a !== slugify(c.title)))],
editPolicy: state.policy,
license: state.license,
tribeId: c.tribeId || null,
encrypted: !!c.tribeId,
author: state.owner,
lastAuthor: tip.author,
createdAt: first.c.createdAt || new Date(first.ts).toISOString(),
updatedAt: c.updatedAt || new Date(tip.ts).toISOString(),
ts: tip.ts,
versionCount: versions.length,
links: extractWikiLinks(body),
versions: versions.map(vn => ({
key: vn.key,
author: vn.author,
ts: vn.ts,
createdAt: vn.c.updatedAt || vn.c.createdAt || new Date(vn.ts).toISOString(),
title: safeText(vn.c.title),
body: safeText(vn.c.body),
tags: normalizeList(vn.c.tags),
summary: safeText(vn.c.summary)
}))
});
}
pages.sort((a, b) => a.createdAt < b.createdAt ? -1 : a.createdAt > b.createdAt ? 1 : 0);
return { pages, rootOf, nodes };
};
const sameScope = (page, tribeId) => String(page.tribeId || '') === String(tribeId || '');
const scopePages = async (pages, tribeId) => {
if (tribeId === 'any') return pages;
if (!tribeId) return pages.filter(p => !p.tribeId);
const chain = tribesModel && typeof tribesModel.getChainIds === 'function'
? await tribesModel.getChainIds(tribeId).catch(() => [tribeId])
: [tribeId];
const set = new Set(chain.map(String));
return pages.filter(p => p.tribeId && set.has(String(p.tribeId)));
};
const decorate = (pages, scoped) => {
const bySlug = new Map();
for (const p of scoped) {
const keys = [p.slug, ...p.aliases];
for (const s of keys) if (s && !bySlug.has(s)) bySlug.set(s, p);
}
const inbound = new Map();
for (const p of scoped) {
for (const s of p.links) {
const target = bySlug.get(s);
if (!target || target.id === p.id) continue;
if (!inbound.has(target.id)) inbound.set(target.id, new Set());
inbound.get(target.id).add(p.id);
}
}
return scoped.map(p => ({
...p,
backlinkCount: (inbound.get(p.id) || new Set()).size,
missingLinks: p.links.filter(s => !bySlug.has(s)),
isLinked: (inbound.get(p.id) || new Set()).size > 0
}));
};
const findInScope = (scoped, idOrSlug) => {
const needle = String(idOrSlug || '').trim();
if (!needle) return null;
if (needle.startsWith('%')) return scoped.find(p => p.id === needle || p.tipId === needle || p.versions.some(v => v.key === needle)) || null;
const slug = slugify(needle);
return scoped.find(p => p.slug === slug) || scoped.find(p => p.aliases.includes(slug)) || null;
};
const publishContent = async (ssbClient, content) => {
let msg = content;
if (content.tribeId) {
const key = await tribeKeyFor(content.tribeId);
if (!key) throw new Error('Missing tribe key — cannot publish encrypted wiki page');
msg = tribeCrypto.wrapMsg({ k: WIKI_TYPE, ...content }, key);
}
return new Promise((resolve, reject) => ssbClient.publish(msg, (err, res) => err ? reject(err) : resolve(res)));
};
const load = async () => {
const ssbClient = await openSsb();
const entries = await readEntries(ssbClient);
const members = await tribeMembersFor(entries);
const idx = memoIndex('wiki', entries, (e) => buildIndex(e, members));
return { ssbClient, idx };
};
const canEdit = async (page, viewer) => {
if (!page) return false;
const owner = String(page.author) === String(viewer);
if (page.editPolicy === 'author') return owner;
if (page.tribeId) {
if (!tribesModel) return owner;
try {
const t = await tribesModel.getTribeById(page.tribeId);
return owner || (Array.isArray(t.members) && t.members.includes(viewer));
} catch (_) { return owner; }
}
if (page.editPolicy === 'tribe') return owner;
return true;
};
return {
WIKI_TYPE,
EDIT_POLICIES,
slugify,
async createPage({ title, body, tags, aliases, editPolicy, license, tribeId = null }) {
const { ssbClient, idx } = await load();
const cleanTitle = safeText(title).slice(0, MAX_TITLE);
if (!cleanTitle) throw new Error('Title required');
const slug = slugify(cleanTitle);
if (!slug) throw new Error('Title required');
const scoped = await scopePages(idx.pages, tribeId);
const existing = scoped.find(p => p.slug === slug);
if (existing) return { key: existing.id, existing: true };
const now = new Date().toISOString();
const content = {
type: WIKI_TYPE,
title: cleanTitle,
slug,
...(await storeBody(ssbClient, body, tribeId)),
tags: normalizeList(tags),
aliases: normalizeList(aliases).map(slugify).filter(Boolean),
editPolicy: normalizePolicy(editPolicy),
...(license ? { license } : {}),
author: ssbClient.id,
createdAt: now,
updatedAt: now,
...(tribeId ? { tribeId } : {})
};
const res = await publishContent(ssbClient, content);
return { key: res.key, existing: false };
},
async updatePage(id, data = {}) {
const { ssbClient, idx } = await load();
const page = findInScope(idx.pages, id);
if (!page) throw new Error('Page not found');
if (!(await canEdit(page, ssbClient.id))) throw new Error('Not allowed to edit');
const now = new Date().toISOString();
const nextTitle = data.title !== undefined ? safeText(data.title).slice(0, MAX_TITLE) : page.title;
const nextLicense = data.license !== undefined && String(page.author) === String(ssbClient.id) ? safeText(data.license) : page.license;
const content = {
type: WIKI_TYPE,
title: nextTitle || page.title,
slug: page.slug,
...(await storeBody(ssbClient, data.body !== undefined ? data.body : page.body, page.tribeId || null)),
tags: data.tags !== undefined ? normalizeList(data.tags) : page.tags,
aliases: data.aliases !== undefined ? normalizeList(data.aliases).map(slugify).filter(Boolean) : page.aliases,
editPolicy: data.editPolicy !== undefined ? normalizePolicy(data.editPolicy) : page.editPolicy,
...(nextLicense ? { license: nextLicense } : {}),
summary: safeText(data.summary).slice(0, 200),
author: ssbClient.id,
createdAt: page.createdAt,
updatedAt: now,
replaces: page.tipId,
...(page.tribeId ? { tribeId: page.tribeId } : {})
};
const res = await publishContent(ssbClient, content);
return { key: res.key, rootId: page.id, author: page.author };
},
async restoreVersion(id, versionKey) {
const { idx } = await load();
const page = findInScope(idx.pages, id);
if (!page) throw new Error('Page not found');
const version = page.versions.find(v => v.key === versionKey);
if (!version) throw new Error('Version not found');
return this.updatePage(page.id, { title: version.title, body: version.body, tags: version.tags, summary: `restore:${versionKey}` });
},
async deletePage(id) {
const { ssbClient, idx } = await load();
const page = findInScope(idx.pages, id);
if (!page) throw new Error('Page not found');
if (String(page.author) !== String(ssbClient.id)) throw new Error('Only the creator can delete');
const deletedAt = new Date().toISOString();
const tomb = { type: 'tombstone', target: page.id, deletedAt, author: ssbClient.id };
let msg = tomb;
if (page.tribeId) {
const key = await tribeKeyFor(page.tribeId);
if (!key) throw new Error('Missing tribe key — cannot delete encrypted wiki page');
msg = tribeCrypto.wrapMsg({ k: 'tombstone', target: page.id, deletedAt, author: ssbClient.id }, key);
}
return new Promise((resolve, reject) => ssbClient.publish(msg, (err, res) => err ? reject(err) : resolve(res)));
},
async getPage(idOrSlug, { tribeId = null } = {}) {
const { ssbClient, idx } = await load();
let scoped = await scopePages(idx.pages, tribeId);
let page = findInScope(scoped, idOrSlug);
if (!page && String(idOrSlug || '').startsWith('%')) {
page = findInScope(idx.pages, idOrSlug);
if (page) scoped = await scopePages(idx.pages, page.tribeId || null);
}
if (!page) return null;
const decorated = decorate(idx.pages, scoped);
const full = decorated.find(p => p.id === page.id) || page;
const bySlug = new Map();
for (const p of scoped) for (const s of [p.slug, ...p.aliases]) if (s && !bySlug.has(s)) bySlug.set(s, p);
const backlinks = decorated.filter(p => p.id !== full.id && p.links.some(s => bySlug.get(s) && bySlug.get(s).id === full.id));
const outbound = [];
const seen = new Set([full.id]);
for (const slug of full.links) {
const target = bySlug.get(slug);
if (!target || seen.has(target.id)) continue;
seen.add(target.id);
outbound.push(decorated.find(p => p.id === target.id) || target);
}
const linkedPages = [...backlinks];
const inList = new Set(backlinks.map(p => p.id));
for (const p of outbound) if (!inList.has(p.id)) linkedPages.push(p);
return { ...full, backlinks, outbound, linkedPages, canEdit: await canEdit(full, ssbClient.id), isOwner: String(full.author) === String(ssbClient.id) };
},
async listPages({ tribeId = null, filter = 'all', q = '', viewerId } = {}) {
const { ssbClient, idx } = await load();
const me = viewerId || ssbClient.id;
const scoped = decorate(idx.pages, await scopePages(idx.pages, tribeId));
let list = scoped;
const f = String(filter || 'all').toLowerCase();
if (f === 'mine') list = list.filter(p => String(p.author) === String(me) || p.versions.some(v => String(v.author) === String(me)));
else if (f === 'recent') list = list.filter(p => Date.now() - p.ts <= RECENT_MS);
else if (f === 'linked') list = list.filter(p => p.isLinked);
const needle = safeText(q).toLowerCase();
if (needle) list = list.filter(p => p.title.toLowerCase().includes(needle) || p.body.toLowerCase().includes(needle) || p.tags.some(t => t.toLowerCase().includes(needle)));
return list.slice().sort((a, b) => b.ts - a.ts);
},
async recentChanges({ tribeId = null } = {}) {
const { idx } = await load();
const scoped = await scopePages(idx.pages, tribeId);
const changes = [];
for (const p of scoped) {
p.versions.forEach((v, i) => changes.push({
pageId: p.id, slug: p.slug, title: v.title || p.title, tribeId: p.tribeId || null,
versionKey: v.key, author: v.author, ts: v.ts, createdAt: v.createdAt, summary: v.summary, index: i + 1, isCreation: i === 0
}));
}
return changes.sort((a, b) => b.ts - a.ts || b.index - a.index).slice(0, MAX_CHANGES);
},
async resolveRootId(id) {
const { idx } = await load();
const page = findInScope(idx.pages, id);
if (!page) throw new Error('Not found');
return page.id;
},
async canEditPage(id, viewer) {
const { ssbClient, idx } = await load();
const page = findInScope(idx.pages, id);
return canEdit(page, viewer || ssbClient.id);
}
};
};
Object.assign(module.exports, { WIKILINK_RE, slugify, linkTarget, extractWikiLinks });