|
|
|
|
@ -4,13 +4,57 @@ require('dotenv').config(); // Cargar variables de entorno desde .env
|
|
|
|
|
|
|
|
|
|
const express = require('express');
|
|
|
|
|
const path = require('path');
|
|
|
|
|
const fs = require('fs');
|
|
|
|
|
const bodyParser = require('body-parser');
|
|
|
|
|
const helmet = require('helmet');
|
|
|
|
|
const multer = require('multer');
|
|
|
|
|
const { MongoClient } = require('mongodb');
|
|
|
|
|
|
|
|
|
|
const app = express();
|
|
|
|
|
const port = process.env.PORT || 3000;
|
|
|
|
|
|
|
|
|
|
// Colección de comparaciones servida al grafo (configurable para conmutar al motor semántico v2)
|
|
|
|
|
const COMPARACIONES_COL = process.env.COMPARACIONES_COL || 'comparaciones';
|
|
|
|
|
|
|
|
|
|
// --- Buzón seguro de colaboradores/periodistas ---------------------------
|
|
|
|
|
// Los envíos se guardan FUERA del webroot (nunca se sirven como estáticos).
|
|
|
|
|
const SUBMISSIONS_DIR = process.env.SUBMISSIONS_DIR
|
|
|
|
|
|| path.join(__dirname, '../../FLUJOS_DATOS/SUBMISSIONS');
|
|
|
|
|
|
|
|
|
|
// Imagenes de Wikipedia ya scrapeadas; se sirven en /wiki-images
|
|
|
|
|
const WIKI_IMAGES_DIR = process.env.WIKI_IMAGES_DIR
|
|
|
|
|
|| path.join(__dirname, '../../FLUJOS_DATOS/IMAGENES/output/wiki_images');
|
|
|
|
|
try { fs.mkdirSync(SUBMISSIONS_DIR, { recursive: true, mode: 0o700 }); } catch (e) { /* noop */ }
|
|
|
|
|
|
|
|
|
|
const submitStorage = multer.diskStorage({
|
|
|
|
|
destination: (req, file, cb) => cb(null, SUBMISSIONS_DIR),
|
|
|
|
|
filename: (req, file, cb) => {
|
|
|
|
|
const safe = (file.originalname || 'archivo')
|
|
|
|
|
.replace(/[^a-zA-Z0-9._-]/g, '_').slice(0, 80);
|
|
|
|
|
cb(null, `${Date.now()}_${Math.random().toString(36).slice(2, 8)}_${safe}`);
|
|
|
|
|
},
|
|
|
|
|
});
|
|
|
|
|
const submitUpload = multer({
|
|
|
|
|
storage: submitStorage,
|
|
|
|
|
limits: { fileSize: 50 * 1024 * 1024, files: 8 }, // 50 MB/archivo, 8 archivos
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
// Rate-limit sencillo en memoria por IP para el buzón (10 envíos/hora)
|
|
|
|
|
const submitHits = new Map();
|
|
|
|
|
function submitRateLimit(req, res, next) {
|
|
|
|
|
const ip = (req.headers['x-forwarded-for'] || '').split(',')[0].trim()
|
|
|
|
|
|| req.socket.remoteAddress || 'desconocida';
|
|
|
|
|
const now = Date.now();
|
|
|
|
|
const windowMs = 60 * 60 * 1000;
|
|
|
|
|
const arr = (submitHits.get(ip) || []).filter((t) => now - t < windowMs);
|
|
|
|
|
if (arr.length >= 10) {
|
|
|
|
|
return res.status(429).json({ ok: false, error: 'Demasiados envíos desde tu conexión. Inténtalo más tarde.' });
|
|
|
|
|
}
|
|
|
|
|
arr.push(now);
|
|
|
|
|
submitHits.set(ip, arr);
|
|
|
|
|
next();
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Helmet completo: X-Powered-By off, XSS protection, nosniff, frameguard, HSTS, etc.
|
|
|
|
|
app.use(helmet());
|
|
|
|
|
|
|
|
|
|
@ -32,12 +76,17 @@ app.use(
|
|
|
|
|
);
|
|
|
|
|
|
|
|
|
|
// Prevenir CSRF: solo aceptar peticiones GET a /api desde el mismo origen
|
|
|
|
|
const ORIGENES_PERMITIDOS = [
|
|
|
|
|
'https://upleaks.org',
|
|
|
|
|
'https://www.upleaks.org',
|
|
|
|
|
`http://localhost:${port}`,
|
|
|
|
|
];
|
|
|
|
|
app.use('/api/', (req, res, next) => {
|
|
|
|
|
const origin = req.headers.origin;
|
|
|
|
|
const referer = req.headers.referer;
|
|
|
|
|
// En producción detrás de nginx no hay Origin en peticiones same-site normales,
|
|
|
|
|
// pero sí lo habría en peticiones cross-origin maliciosas
|
|
|
|
|
if (origin && origin !== `https://theflows.net` && origin !== `http://localhost:${port}`) {
|
|
|
|
|
if (origin && !ORIGENES_PERMITIDOS.includes(origin)) {
|
|
|
|
|
return res.status(403).json({ error: 'Origen no permitido' });
|
|
|
|
|
}
|
|
|
|
|
next();
|
|
|
|
|
@ -54,6 +103,56 @@ app.use(bodyParser.json({ limit: '10kb' }));
|
|
|
|
|
|
|
|
|
|
// **Definir primero las rutas de la API**
|
|
|
|
|
|
|
|
|
|
// --- Proxy de imagenes para las miniaturas del grafo ----------------------
|
|
|
|
|
// Sirve same-origin las og:image remotas de las noticias: WebGL/TextureLoader
|
|
|
|
|
// no puede texturizar imagenes cross-origin sin CORS. Endpoint PUBLICO, asi que
|
|
|
|
|
// lleva proteccion anti-SSRF (solo http(s), nunca IPs privadas/loopback).
|
|
|
|
|
const _dns = require('dns').promises;
|
|
|
|
|
const _net = require('net');
|
|
|
|
|
function _ipPrivada(ip) {
|
|
|
|
|
if (_net.isIPv4(ip)) {
|
|
|
|
|
const p = ip.split('.').map(Number);
|
|
|
|
|
return p[0] === 10 || p[0] === 127 || p[0] === 0
|
|
|
|
|
|| (p[0] === 169 && p[1] === 254)
|
|
|
|
|
|| (p[0] === 172 && p[1] >= 16 && p[1] <= 31)
|
|
|
|
|
|| (p[0] === 192 && p[1] === 168);
|
|
|
|
|
}
|
|
|
|
|
if (_net.isIPv6(ip)) {
|
|
|
|
|
const l = ip.toLowerCase();
|
|
|
|
|
return l === '::1' || l.startsWith('fc') || l.startsWith('fd')
|
|
|
|
|
|| l.startsWith('fe80') || l.startsWith('::ffff:');
|
|
|
|
|
}
|
|
|
|
|
return true; // desconocido -> bloquear
|
|
|
|
|
}
|
|
|
|
|
app.get('/api/img', async (req, res) => {
|
|
|
|
|
try {
|
|
|
|
|
const raw = req.query.u;
|
|
|
|
|
if (!raw || typeof raw !== 'string' || raw.length > 2048) return res.status(400).end();
|
|
|
|
|
let u;
|
|
|
|
|
try { u = new URL(raw); } catch (e) { return res.status(400).end(); }
|
|
|
|
|
if (u.protocol !== 'http:' && u.protocol !== 'https:') return res.status(400).end();
|
|
|
|
|
let dir;
|
|
|
|
|
try { dir = await _dns.lookup(u.hostname); } catch (e) { return res.status(502).end(); }
|
|
|
|
|
if (_ipPrivada(dir.address)) return res.status(403).end();
|
|
|
|
|
const ctrl = new AbortController();
|
|
|
|
|
const t = setTimeout(() => ctrl.abort(), 6000);
|
|
|
|
|
let r;
|
|
|
|
|
try {
|
|
|
|
|
r = await fetch(u.href, { signal: ctrl.signal, redirect: 'follow',
|
|
|
|
|
headers: { 'User-Agent': 'Mozilla/5.0 (compatible; upleaks-img/1.0)' } });
|
|
|
|
|
} finally { clearTimeout(t); }
|
|
|
|
|
const ct = r.headers.get('content-type') || '';
|
|
|
|
|
if (!r.ok || !ct.startsWith('image/')) return res.status(404).end();
|
|
|
|
|
const buf = Buffer.from(await r.arrayBuffer());
|
|
|
|
|
if (buf.length > 8 * 1024 * 1024) return res.status(413).end();
|
|
|
|
|
res.set('Content-Type', ct);
|
|
|
|
|
res.set('Cache-Control', 'public, max-age=86400');
|
|
|
|
|
return res.end(buf);
|
|
|
|
|
} catch (e) {
|
|
|
|
|
return res.status(500).end();
|
|
|
|
|
}
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
app.get('/api/data', async (req, res) => {
|
|
|
|
|
console.log('Solicitud recibida en /api/data');
|
|
|
|
|
try {
|
|
|
|
|
@ -69,7 +168,7 @@ app.get('/api/data', async (req, res) => {
|
|
|
|
|
const torrentsCollection = db.collection('torrents');
|
|
|
|
|
const imagenesCollection = db.collection('imagenes'); // analizadas por Qwen
|
|
|
|
|
const imagenesWikiCollection = db.collection('imagenes_wiki'); // scrapeadas (fallback)
|
|
|
|
|
const comparacionesCollection = db.collection('comparaciones');
|
|
|
|
|
const comparacionesCollection = db.collection(COMPARACIONES_COL);
|
|
|
|
|
|
|
|
|
|
// ── Sanitización y validación de parámetros ──────────────────────────────
|
|
|
|
|
// Rechaza cualquier valor que no sea string (bloquea operator injection)
|
|
|
|
|
@ -134,6 +233,10 @@ app.get('/api/data', async (req, res) => {
|
|
|
|
|
res.status(400).json({ error: 'Rango de fechas inválido' });
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
// `new Date('2026-08-23')` es 00:00:00Z, asi que un `$lte` dejaba fuera el
|
|
|
|
|
// dia final entero (~2000 noticias/semana, justo las mas recientes).
|
|
|
|
|
// Guardamos el instante siguiente y filtramos con `$lt`.
|
|
|
|
|
fechaLte = new Date(fechaLte.getTime() + 24 * 60 * 60 * 1000);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Validar nodos: entero positivo entre 1 y 500
|
|
|
|
|
@ -149,13 +252,33 @@ app.get('/api/data', async (req, res) => {
|
|
|
|
|
const nodesQuery = { tema };
|
|
|
|
|
if (subtematica) nodesQuery.subtema = subtematica;
|
|
|
|
|
if (regexKw) nodesQuery.texto = { $regex: regexKw, $options: 'i' };
|
|
|
|
|
if (fechaGte) nodesQuery.fecha = { $gte: fechaGte, $lte: fechaLte };
|
|
|
|
|
// El filtro de fecha (modo "semanal") y el orden por frescura se aplican SOLO
|
|
|
|
|
// a noticias: wikipedia/torrents/imagenes no tienen campo `fecha`, y meterles
|
|
|
|
|
// el rango las vaciaria del grafo (perdiendo las conexiones noticia<->leak).
|
|
|
|
|
//
|
|
|
|
|
// Dos modos, y tienen que verse distintos:
|
|
|
|
|
// Semanal (hay rango) -> solo noticias FECHADAS dentro de la ventana,
|
|
|
|
|
// ordenadas por frescura. Las de `fecha: null`
|
|
|
|
|
// quedan fuera (un $gte nunca casa con null).
|
|
|
|
|
// Todas (sin rango) -> muestra aleatoria de TODO el archivo, incluidas
|
|
|
|
|
// las de `fecha: null`. Con sort({fecha:-1}) +
|
|
|
|
|
// limit(N) las 100 mas nuevas eran siempre las de
|
|
|
|
|
// esta semana, asi que los dos modos devolvian el
|
|
|
|
|
// mismo grafo byte a byte.
|
|
|
|
|
const modoSemanal = Boolean(fechaGte);
|
|
|
|
|
const noticiasQuery = { ...nodesQuery };
|
|
|
|
|
if (modoSemanal) noticiasQuery.fecha = { $gte: fechaGte, $lt: fechaLte };
|
|
|
|
|
// Imágenes: proporcional a nodosLimit (1/3), mínimo 3 para siempre mostrar algo
|
|
|
|
|
const imagenesLimit = Math.max(3, Math.floor(nodosLimit / 3));
|
|
|
|
|
|
|
|
|
|
const [wikipediaNodes, noticiasNodes, torrentsNodes, imagenesNodes, imagenesWikiNodes] = await Promise.all([
|
|
|
|
|
wikipediaCollection.find(nodesQuery).limit(nodosLimit).toArray(),
|
|
|
|
|
noticiasCollection.find(nodesQuery).limit(nodosLimit).toArray(),
|
|
|
|
|
modoSemanal
|
|
|
|
|
? noticiasCollection.find(noticiasQuery).sort({ fecha: -1 }).limit(nodosLimit).toArray()
|
|
|
|
|
: noticiasCollection.aggregate([
|
|
|
|
|
{ $match: noticiasQuery },
|
|
|
|
|
{ $sample: { size: nodosLimit } },
|
|
|
|
|
]).toArray(),
|
|
|
|
|
torrentsCollection.find(nodesQuery).limit(nodosLimit).toArray(),
|
|
|
|
|
imagenesCollection.find(nodesQuery).limit(imagenesLimit).toArray(),
|
|
|
|
|
imagenesWikiCollection.find(nodesQuery).limit(imagenesLimit).toArray(),
|
|
|
|
|
@ -174,11 +297,14 @@ app.get('/api/data', async (req, res) => {
|
|
|
|
|
content: result.texto || '',
|
|
|
|
|
fecha: result.fecha || '',
|
|
|
|
|
fuente: result.fuente || result.autor || result.source || '',
|
|
|
|
|
imagen_url: result.imagen_url || '',
|
|
|
|
|
type: 'texto',
|
|
|
|
|
}));
|
|
|
|
|
|
|
|
|
|
// Nodos de imagen — image_url construida desde image_path absoluto
|
|
|
|
|
const WIKI_IMAGES_BASE = '/var/www/theflows.net/flujos/FLUJOS_DATOS/IMAGENES/output/wiki_images/';
|
|
|
|
|
const WIKI_IMAGES_BASE = WIKI_IMAGES_DIR.endsWith(path.sep)
|
|
|
|
|
? WIKI_IMAGES_DIR
|
|
|
|
|
: WIKI_IMAGES_DIR + path.sep;
|
|
|
|
|
const imgFormattedNodes = imgNodes.map((result) => {
|
|
|
|
|
let relativePath = null;
|
|
|
|
|
if (result.image_path && result.image_path.startsWith(WIKI_IMAGES_BASE)) {
|
|
|
|
|
@ -372,6 +498,319 @@ app.get('/api/stats', async (req, res) => {
|
|
|
|
|
}
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
// **Buzón seguro: recepción de material de colaboradores/periodistas**
|
|
|
|
|
const TEMAS_VALIDOS = new Set([
|
|
|
|
|
'guerra global', 'inteligencia y seguridad', 'cambio climático',
|
|
|
|
|
'economía y corporaciones', 'demografía y sociedad', 'otros',
|
|
|
|
|
]);
|
|
|
|
|
|
|
|
|
|
// Página HTML de resultado (para envíos desde el formulario nativo, sin JavaScript)
|
|
|
|
|
function renderSubmitPage(payload) {
|
|
|
|
|
const ok = payload.ok;
|
|
|
|
|
const inner = ok
|
|
|
|
|
? `<div class="tag ok">Envío recibido</div>
|
|
|
|
|
<h1>Gracias. Tu material está a salvo.</h1>
|
|
|
|
|
<p>Guarda este código de referencia. Es tu único identificador y no lo compartas por canales inseguros:</p>
|
|
|
|
|
<div class="ref">${payload.ref}</div>
|
|
|
|
|
<p class="small">No hemos registrado tu IP. Si dejaste un contacto, te escribiremos por ahí.</p>`
|
|
|
|
|
: `<div class="tag err">No se pudo enviar</div>
|
|
|
|
|
<h1>Algo ha fallado</h1>
|
|
|
|
|
<p>${(payload.error || 'Inténtalo de nuevo.').replace(/[<>&]/g, '')}</p>`;
|
|
|
|
|
return `<!DOCTYPE html><html lang="es"><head><meta charset="UTF-8">
|
|
|
|
|
<meta name="viewport" content="width=device-width,initial-scale=1"><meta name="robots" content="noindex">
|
|
|
|
|
<title>UP-LEAKS // ${ok ? 'envío recibido' : 'error'}</title>
|
|
|
|
|
<style>
|
|
|
|
|
@font-face{font-family:'Fira Code';src:url('/fonts/fira-code-400.woff2') format('woff2');font-weight:400}
|
|
|
|
|
@font-face{font-family:'Fira Code';src:url('/fonts/fira-code-700.woff2') format('woff2');font-weight:700}
|
|
|
|
|
*{box-sizing:border-box;margin:0}
|
|
|
|
|
body{min-height:100vh;display:flex;align-items:center;justify-content:center;padding:24px;
|
|
|
|
|
font-family:'Fira Code',monospace;color:#d7f5d0;text-align:center;
|
|
|
|
|
background:linear-gradient(rgba(0,0,0,.86),rgba(0,8,0,.95)),url('/images/flujos6.jpg') center/cover fixed;}
|
|
|
|
|
.card{max-width:560px;background:rgba(6,10,6,.75);border:1px solid rgba(57,255,20,.35);
|
|
|
|
|
backdrop-filter:blur(8px);padding:44px 38px;box-shadow:0 0 60px -20px rgba(57,255,20,.5)}
|
|
|
|
|
.tag{display:inline-block;font-size:12px;letter-spacing:.18em;text-transform:uppercase;padding:4px 12px;border:1px solid;margin-bottom:18px}
|
|
|
|
|
.tag.ok{color:#39ff14;border-color:#39ff14}.tag.err{color:#ff2d2d;border-color:#ff2d2d}
|
|
|
|
|
h1{font-size:clamp(24px,4vw,32px);margin-bottom:14px;font-weight:700;line-height:1.15}
|
|
|
|
|
p{color:#8fb98a;font-size:15px;line-height:1.6;margin-bottom:14px}.small{font-size:13px;color:#6f9a6a}
|
|
|
|
|
.ref{font-size:22px;font-weight:700;color:#39ff14;letter-spacing:.08em;text-shadow:0 0 12px rgba(57,255,20,.6);
|
|
|
|
|
border:1px dashed rgba(57,255,20,.5);padding:14px;margin:20px 0}
|
|
|
|
|
.btn{display:inline-block;margin-top:16px;color:#39ff14;text-decoration:none;border:1px solid #39ff14;
|
|
|
|
|
padding:12px 28px;text-transform:uppercase;letter-spacing:.06em;font-weight:700;font-size:13px;transition:.18s}
|
|
|
|
|
.btn:hover{background:#39ff14;color:#000;box-shadow:0 0 18px -2px rgba(57,255,20,.7)}
|
|
|
|
|
</style></head><body><div class="card">${inner}
|
|
|
|
|
<a class="btn" href="/journalist.html">${ok ? 'volver al buzón' : 'reintentar'}</a></div></body></html>`;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function sendSubmitResult(req, res, status, payload) {
|
|
|
|
|
const wantsHtml = (req.body && req.body.render === 'html') || (req.headers.accept || '').includes('text/html');
|
|
|
|
|
if (wantsHtml) return res.status(status).type('html').send(renderSubmitPage(payload));
|
|
|
|
|
return res.status(status).json(payload);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
app.post('/api/submit', submitRateLimit, (req, res) => {
|
|
|
|
|
submitUpload.array('adjuntos', 8)(req, res, async (err) => {
|
|
|
|
|
if (err) {
|
|
|
|
|
const msg = err.code === 'LIMIT_FILE_SIZE'
|
|
|
|
|
? 'Algún archivo supera el límite de 50 MB.'
|
|
|
|
|
: (err.code === 'LIMIT_FILE_COUNT' ? 'Máximo 8 archivos por envío.' : 'Error al subir los archivos.');
|
|
|
|
|
return sendSubmitResult(req, res, 400, { ok: false, error: msg });
|
|
|
|
|
}
|
|
|
|
|
try {
|
|
|
|
|
// Honeypot anti-bots: si el campo oculto viene relleno, descartamos en silencio.
|
|
|
|
|
if (req.body && req.body.website) {
|
|
|
|
|
return sendSubmitResult(req, res, 200, { ok: true, ref: 'RECIBIDO' });
|
|
|
|
|
}
|
|
|
|
|
const mensaje = (req.body.mensaje || '').toString().slice(0, 20000).trim();
|
|
|
|
|
let tema = (req.body.tema || 'otros').toString().trim();
|
|
|
|
|
if (!TEMAS_VALIDOS.has(tema)) tema = 'otros';
|
|
|
|
|
const contacto = (req.body.contacto || '').toString().slice(0, 400).trim();
|
|
|
|
|
const files = req.files || [];
|
|
|
|
|
|
|
|
|
|
if (!mensaje && files.length === 0) {
|
|
|
|
|
return sendSubmitResult(req, res, 400, { ok: false, error: 'Escribe un mensaje o adjunta al menos un archivo.' });
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
const ref = 'UPL-' + Date.now().toString(36).toUpperCase()
|
|
|
|
|
+ '-' + Math.random().toString(36).slice(2, 6).toUpperCase();
|
|
|
|
|
|
|
|
|
|
const doc = {
|
|
|
|
|
ref,
|
|
|
|
|
tema,
|
|
|
|
|
mensaje,
|
|
|
|
|
contacto,
|
|
|
|
|
archivos: files.map((f) => ({
|
|
|
|
|
nombre_original: f.originalname,
|
|
|
|
|
guardado: path.basename(f.path),
|
|
|
|
|
bytes: f.size,
|
|
|
|
|
mimetype: f.mimetype,
|
|
|
|
|
})),
|
|
|
|
|
estado: 'nuevo',
|
|
|
|
|
creado: new Date(),
|
|
|
|
|
};
|
|
|
|
|
// No registramos IP ni contenido en logs por privacidad de la fuente.
|
|
|
|
|
if (db) await db.collection('submissions').insertOne(doc);
|
|
|
|
|
return sendSubmitResult(req, res, 200, { ok: true, ref });
|
|
|
|
|
} catch (e) {
|
|
|
|
|
console.error('Error en /api/submit:', e.message);
|
|
|
|
|
return sendSubmitResult(req, res, 500, { ok: false, error: 'No se pudo procesar el envío. Inténtalo de nuevo.' });
|
|
|
|
|
}
|
|
|
|
|
});
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
// **Panel de administración: lectura de envíos (fail-closed)**
|
|
|
|
|
const crypto = require('crypto');
|
|
|
|
|
const ADMIN_TOKEN_FILE = path.join(__dirname, '../../FLUJOS_DATOS/.admin_token');
|
|
|
|
|
|
|
|
|
|
function getAdminToken() {
|
|
|
|
|
if (process.env.ADMIN_TOKEN && process.env.ADMIN_TOKEN.trim()) return process.env.ADMIN_TOKEN.trim();
|
|
|
|
|
try {
|
|
|
|
|
const t = fs.readFileSync(ADMIN_TOKEN_FILE, 'utf8').trim();
|
|
|
|
|
return t || null;
|
|
|
|
|
} catch (e) { return null; }
|
|
|
|
|
}
|
|
|
|
|
function safeEqual(a, b) {
|
|
|
|
|
const ba = Buffer.from(String(a)); const bb = Buffer.from(String(b));
|
|
|
|
|
if (ba.length !== bb.length) return false;
|
|
|
|
|
return crypto.timingSafeEqual(ba, bb);
|
|
|
|
|
}
|
|
|
|
|
function requireAdmin(req, res, next) {
|
|
|
|
|
const token = getAdminToken();
|
|
|
|
|
if (!token) return res.status(503).json({ ok: false, error: 'Panel no configurado: falta ADMIN_TOKEN o FLUJOS_DATOS/.admin_token' });
|
|
|
|
|
const auth = req.headers.authorization || '';
|
|
|
|
|
const provided = auth.startsWith('Bearer ') ? auth.slice(7) : '';
|
|
|
|
|
if (!provided || !safeEqual(provided, token)) {
|
|
|
|
|
return res.status(401).json({ ok: false, error: 'No autorizado' });
|
|
|
|
|
}
|
|
|
|
|
next();
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Listado de envíos (metadatos + mensaje). Más recientes primero.
|
|
|
|
|
app.get('/api/submissions', requireAdmin, async (req, res) => {
|
|
|
|
|
try {
|
|
|
|
|
if (!db) return res.status(503).json({ ok: false, error: 'BD no disponible' });
|
|
|
|
|
const limit = Math.min(parseInt(req.query.limit, 10) || 100, 500);
|
|
|
|
|
const docs = await db.collection('submissions')
|
|
|
|
|
.find({}, { projection: { _id: 0 } })
|
|
|
|
|
.sort({ creado: -1 }).limit(limit).toArray();
|
|
|
|
|
res.json({ ok: true, total: docs.length, submissions: docs });
|
|
|
|
|
} catch (e) {
|
|
|
|
|
console.error('Error /api/submissions:', e.message);
|
|
|
|
|
res.status(500).json({ ok: false, error: 'Error al leer los envíos' });
|
|
|
|
|
}
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
// Marcar un envío como revisado / archivado
|
|
|
|
|
app.post('/api/submissions/:ref/estado', requireAdmin, express.json(), async (req, res) => {
|
|
|
|
|
try {
|
|
|
|
|
if (!db) return res.status(503).json({ ok: false, error: 'BD no disponible' });
|
|
|
|
|
const estado = ['nuevo', 'revisado', 'archivado'].includes((req.body || {}).estado) ? req.body.estado : null;
|
|
|
|
|
if (!estado) return res.status(400).json({ ok: false, error: 'estado inválido' });
|
|
|
|
|
const r = await db.collection('submissions').updateOne({ ref: req.params.ref }, { $set: { estado } });
|
|
|
|
|
res.json({ ok: r.matchedCount > 0, estado });
|
|
|
|
|
} catch (e) {
|
|
|
|
|
res.status(500).json({ ok: false, error: 'Error al actualizar' });
|
|
|
|
|
}
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
// Descarga de un adjunto (valida basename contra traversal; sirve como adjunto)
|
|
|
|
|
app.get('/api/submissions/file/:guardado', requireAdmin, (req, res) => {
|
|
|
|
|
const name = path.basename(req.params.guardado || '');
|
|
|
|
|
if (!name || name.includes('..') || name.startsWith('.')) return res.status(400).json({ ok: false, error: 'nombre inválido' });
|
|
|
|
|
const full = path.join(SUBMISSIONS_DIR, name);
|
|
|
|
|
if (!full.startsWith(SUBMISSIONS_DIR) || !fs.existsSync(full)) return res.status(404).json({ ok: false, error: 'no encontrado' });
|
|
|
|
|
res.setHeader('Content-Type', 'application/octet-stream');
|
|
|
|
|
res.setHeader('Content-Disposition', `attachment; filename="${name}"`);
|
|
|
|
|
res.setHeader('X-Content-Type-Options', 'nosniff');
|
|
|
|
|
fs.createReadStream(full).pipe(res);
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
// **Monitor del sistema: procesos, servicios y progreso de procesamiento (admin)**
|
|
|
|
|
const os = require('os');
|
|
|
|
|
const { exec } = require('child_process');
|
|
|
|
|
function sh(cmd, timeout = 4500) {
|
|
|
|
|
return new Promise((resolve) => { exec(cmd, { timeout }, (e, out) => resolve(e ? '' : String(out))); });
|
|
|
|
|
}
|
|
|
|
|
const SYS_SERVICES = ['mongod', 'flujos-stats', 'coconews-api', 'coconews-ingestor',
|
|
|
|
|
'coconews-translator', 'coconews-ner', 'coconews-embedder', 'flujos-pipeline'];
|
|
|
|
|
const SYS_MATCHERS = [
|
|
|
|
|
{ key: 'mongod', label: 'MongoDB', re: /mongod(\s|$)/ },
|
|
|
|
|
{ key: 'flujos-stats', label: 'FLUJOS · API del grafo', re: /FLUJOS_APP\.js/ },
|
|
|
|
|
{ key: 'coconews-api', label: 'Coconews · API', re: /coconews\/api\/server\.js/ },
|
|
|
|
|
{ key: 'ingestor', label: 'Ingestor RSS', re: /ingestor\.py/ },
|
|
|
|
|
{ key: 'translator', label: 'Traductor (ES)', re: /translator\.py/ },
|
|
|
|
|
{ key: 'ner', label: 'NER · entidades', re: /ner\.py/ },
|
|
|
|
|
{ key: 'embedder', label: 'Embedder en vivo', re: /workers\/embedder\.py/ },
|
|
|
|
|
{ key: 'backfill', label: 'Backfill de embeddings', re: /backfill_embeddings\.py/ },
|
|
|
|
|
{ key: 'motor', label: 'Comparación semántica (v2)', re: /motor_comparacion\.py/ },
|
|
|
|
|
{ key: 'pipeline', label: 'Pipeline maestro', re: /pipeline_maestro\.py/ },
|
|
|
|
|
{ key: 'mongolo', label: 'Comparación (word-overlap, viejo)', re: /pipeline_mongolo\.py/ },
|
|
|
|
|
];
|
|
|
|
|
|
|
|
|
|
app.get('/api/system', requireAdmin, async (req, res) => {
|
|
|
|
|
try {
|
|
|
|
|
const [psOut, actOut, timerOut, svcShow, timerEnabled] = await Promise.all([
|
|
|
|
|
sh('ps -eo pid,pcpu,pmem,rss,etimes,args --sort=-pcpu --no-headers'),
|
|
|
|
|
sh('systemctl is-active ' + SYS_SERVICES.join(' ')),
|
|
|
|
|
sh('systemctl show flujos-pipeline.timer -p NextElapseUSecRealtime -p LastTriggerUSec'),
|
|
|
|
|
sh('systemctl show flujos-pipeline.service -p ActiveState -p SubState'),
|
|
|
|
|
sh('systemctl is-enabled flujos-pipeline.timer'),
|
|
|
|
|
]);
|
|
|
|
|
const parseKV = (s) => { const o = {}; (s || '').split('\n').forEach((l) => { const i = l.indexOf('='); if (i > 0) o[l.slice(0, i)] = l.slice(i + 1).trim(); }); return o; };
|
|
|
|
|
|
|
|
|
|
const actLines = actOut.trim().split('\n');
|
|
|
|
|
const services = SYS_SERVICES.map((s, i) => ({ name: s, active: (actLines[i] || '').trim() === 'active' }));
|
|
|
|
|
|
|
|
|
|
const procesos = [];
|
|
|
|
|
const seen = new Set();
|
|
|
|
|
psOut.split('\n').forEach((line) => {
|
|
|
|
|
const m = line.trim().match(/^(\d+)\s+([\d.]+)\s+([\d.]+)\s+(\d+)\s+(\d+)\s+(.*)$/);
|
|
|
|
|
if (!m) return;
|
|
|
|
|
const args = m[6];
|
|
|
|
|
for (const mt of SYS_MATCHERS) {
|
|
|
|
|
if (mt.re.test(args)) {
|
|
|
|
|
procesos.push({ key: mt.key, label: mt.label, pid: +m[1], cpu: +m[2], mem: +m[3], rss: (+m[4]) * 1024, etimes: +m[5] });
|
|
|
|
|
seen.add(mt.key);
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
let progress = null;
|
|
|
|
|
let comparacion = null;
|
|
|
|
|
if (db) {
|
|
|
|
|
const [torrTotal, torrEmb, torrProc, notiTotal, notiEmb, chunks, feedsAct, feedsFallo, subsNuevos, lastNoti] = await Promise.all([
|
|
|
|
|
db.collection('torrents').estimatedDocumentCount(),
|
|
|
|
|
db.collection('torrents').countDocuments({ embedding: { $exists: true } }), // vectores reales
|
|
|
|
|
db.collection('torrents').countDocuments({ procesado_embedding: true }), // procesados (incluye no-aptos)
|
|
|
|
|
db.collection('noticias').estimatedDocumentCount(),
|
|
|
|
|
db.collection('noticias').countDocuments({ embedding: { $exists: true } }),
|
|
|
|
|
db.collection('embeddings_chunks').estimatedDocumentCount(),
|
|
|
|
|
db.collection('coconews_feeds').countDocuments({ activo: true }),
|
|
|
|
|
db.collection('coconews_feeds').countDocuments({ fallos: { $gt: 0 } }),
|
|
|
|
|
db.collection('submissions').countDocuments({ estado: 'nuevo' }),
|
|
|
|
|
db.collection('noticias').find({}).sort({ _id: -1 }).limit(1).project({ fecha: 1, _id: 0 }).toArray(),
|
|
|
|
|
]);
|
|
|
|
|
progress = { torrTotal, torrEmb, torrProc, torrNoApto: Math.max(0, torrProc - torrEmb),
|
|
|
|
|
notiTotal, notiEmb, chunks, feedsAct, feedsFallo, subsNuevos,
|
|
|
|
|
lastNoti: (lastNoti[0] && lastNoti[0].fecha) || null };
|
|
|
|
|
|
|
|
|
|
// --- Comparación: qué colección sirve el front (v1 word-overlap vs v2 semántica) ---
|
|
|
|
|
const [v1Aristas, v2Aristas] = await Promise.all([
|
|
|
|
|
db.collection('comparaciones').estimatedDocumentCount(),
|
|
|
|
|
db.collection('comparaciones_v2').estimatedDocumentCount(),
|
|
|
|
|
]);
|
|
|
|
|
let v2Eventos = 0, v2Muestra = null;
|
|
|
|
|
if (v2Aristas > 0) {
|
|
|
|
|
[v2Eventos, v2Muestra] = await Promise.all([
|
|
|
|
|
db.collection('comparaciones_v2').countDocuments({ posible_evento: true }),
|
|
|
|
|
db.collection('comparaciones_v2').find({}).sort({ _id: -1 }).limit(1).project({ metodo: 1, modelo: 1, _id: 0 }).toArray().then((a) => a[0] || null),
|
|
|
|
|
]);
|
|
|
|
|
}
|
|
|
|
|
comparacion = {
|
|
|
|
|
activa: COMPARACIONES_COL,
|
|
|
|
|
v2Activa: COMPARACIONES_COL === 'comparaciones_v2',
|
|
|
|
|
v1Aristas, v2Aristas, v2Eventos,
|
|
|
|
|
metodo: (v2Muestra && v2Muestra.metodo) || null,
|
|
|
|
|
modelo: (v2Muestra && v2Muestra.modelo) || null,
|
|
|
|
|
};
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// --- Pipeline maestro: reconstruir ejecuciones desde pipeline_log ---
|
|
|
|
|
const PIPE_FASES = ['scraping', 'analisis', 'comparacion'];
|
|
|
|
|
const PIPE_LABEL = { scraping: 'Scraping', analisis: 'Análisis', comparacion: 'Comparación' };
|
|
|
|
|
let pipelineMaestro = null;
|
|
|
|
|
if (db) {
|
|
|
|
|
const logs = await db.collection('pipeline_log').find({}).sort({ inicio: 1 }).toArray();
|
|
|
|
|
const runs = []; let cur = null;
|
|
|
|
|
logs.forEach((l) => {
|
|
|
|
|
if (l.fase === 'scraping' || !cur) { if (cur) runs.push(cur); cur = { inicio: l.inicio, fin: l.fin, fases: {} }; }
|
|
|
|
|
cur.fases[l.fase] = { estado: l.estado, inicio: l.inicio, fin: l.fin, stats: l.stats || {} };
|
|
|
|
|
if (l.fin && (!cur.fin || new Date(l.fin) > new Date(cur.fin))) cur.fin = l.fin;
|
|
|
|
|
});
|
|
|
|
|
if (cur) runs.push(cur);
|
|
|
|
|
const currentRun = runs[runs.length - 1] || null;
|
|
|
|
|
const finished = runs.filter((r) => r.fases.comparacion && r.fases.comparacion.fin);
|
|
|
|
|
const lastFinished = finished[finished.length - 1] || null;
|
|
|
|
|
const nowMs = Date.now();
|
|
|
|
|
const fasesArr = PIPE_FASES.map((f) => {
|
|
|
|
|
const e = currentRun && currentRun.fases[f];
|
|
|
|
|
const durSec = e ? (e.fin ? (new Date(e.fin) - new Date(e.inicio)) / 1000 : (nowMs - new Date(e.inicio)) / 1000) : null;
|
|
|
|
|
return { fase: f, label: PIPE_LABEL[f], estado: e ? e.estado : 'pendiente', durSec, stats: (e && e.stats) || {} };
|
|
|
|
|
});
|
|
|
|
|
const svcv = parseKV(svcShow);
|
|
|
|
|
const timerv = parseKV(timerOut);
|
|
|
|
|
const nextU = parseInt(timerv.NextElapseUSecRealtime, 10);
|
|
|
|
|
const lastU = parseInt(timerv.LastTriggerUSec, 10);
|
|
|
|
|
let nextRun = null;
|
|
|
|
|
if (nextU > 0) nextRun = Math.round(nextU / 1000);
|
|
|
|
|
else if (lastU > 0) nextRun = Math.round(lastU / 1000) + 12 * 3600 * 1000; // cada 12 h
|
|
|
|
|
const running = svcv.SubState === 'start' || svcv.ActiveState === 'activating' || fasesArr.some((f) => f.estado === 'en_progreso');
|
|
|
|
|
pipelineMaestro = {
|
|
|
|
|
fases: fasesArr,
|
|
|
|
|
runInicio: currentRun ? currentRun.inicio : null,
|
|
|
|
|
runElapsedSec: currentRun ? (nowMs - new Date(currentRun.inicio)) / 1000 : null,
|
|
|
|
|
lastDurSec: lastFinished ? (new Date(lastFinished.fin) - new Date(lastFinished.inicio)) / 1000 : null,
|
|
|
|
|
lastFin: lastFinished ? lastFinished.fin : null,
|
|
|
|
|
nextRun, running, cadencia: 'cada 12 h',
|
|
|
|
|
habilitado: (timerEnabled || '').trim() === 'enabled',
|
|
|
|
|
estado: fasesArr.some((f) => f.estado === 'error') ? 'error' : (running ? 'en_progreso' : 'idle'),
|
|
|
|
|
};
|
|
|
|
|
}
|
|
|
|
|
const nextPipeline = pipelineMaestro ? pipelineMaestro.nextRun : null;
|
|
|
|
|
|
|
|
|
|
res.json({
|
|
|
|
|
ok: true,
|
|
|
|
|
host: { load: os.loadavg(), cores: os.cpus().length, totalmem: os.totalmem(), freemem: os.freemem(), uptime: os.uptime() },
|
|
|
|
|
services, procesos, progress, nextPipeline, comparacion,
|
|
|
|
|
pipelines: { maestro: pipelineMaestro },
|
|
|
|
|
now: Date.now(),
|
|
|
|
|
});
|
|
|
|
|
} catch (e) {
|
|
|
|
|
console.error('/api/system:', e.message);
|
|
|
|
|
res.status(500).json({ ok: false, error: 'No se pudo leer el estado del sistema' });
|
|
|
|
|
}
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
// **Luego, definir las rutas de las páginas principales**
|
|
|
|
|
// Rutas para las páginas principales
|
|
|
|
|
app.get('/', (req, res) => {
|
|
|
|
|
@ -405,9 +844,7 @@ app.get('/eco-corp.html', (req, res) => {
|
|
|
|
|
// **Después, configurar el middleware de archivos estáticos**
|
|
|
|
|
|
|
|
|
|
// Imágenes scrapeadas de Wikipedia
|
|
|
|
|
app.use('/wiki-images', express.static(
|
|
|
|
|
'/var/www/theflows.net/flujos/FLUJOS_DATOS/IMAGENES/output/wiki_images'
|
|
|
|
|
));
|
|
|
|
|
app.use('/wiki-images', express.static(WIKI_IMAGES_DIR));
|
|
|
|
|
|
|
|
|
|
app.use(express.static(path.join(__dirname, '../VISUALIZACION/public')));
|
|
|
|
|
|
|
|
|
|
|
hacklab