Compare commits
8 commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
d9ea78b8a7 | ||
|
|
10f0555c46 | ||
|
|
b3bf3d7a7f | ||
|
|
ec839b5b54 | ||
|
|
e3c682a36f | ||
|
|
ab3b0b53c5 | ||
|
|
00c0254e6c | ||
|
|
10d51c3c52 |
173 changed files with 4376 additions and 21789 deletions
|
|
@ -1,39 +0,0 @@
|
|||
#!/bin/bash
|
||||
# Hook para docker compose - Se ejecuta automáticamente
|
||||
# Copia este archivo al inicio de tu .bashrc o .bash_profile
|
||||
|
||||
# Función de validación automática
|
||||
validate_credentials() {
|
||||
# Verificar .env
|
||||
if [ ! -f .env ]; then
|
||||
echo "🔑 Generando credenciales..."
|
||||
./generate_secure_credentials.sh --force
|
||||
return $?
|
||||
fi
|
||||
|
||||
# Verificar contraseñas vacías
|
||||
for var in POSTGRES_PASSWORD REDIS_PASSWORD DB_PASS; do
|
||||
val=$(grep "^${var}=" .env | cut -d'=' -f2 | tr -d ' ')
|
||||
if [ -z "$val" ] || [ "$val" = "change_*" ]; then
|
||||
echo "🔑 Generando credenciales para $var..."
|
||||
./generate_secure_credentials.sh --force
|
||||
return $?
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
# Alias para docker compose con validación automática
|
||||
alias docker="docker"
|
||||
alias "docker compose"="docker compose --env-file .env"
|
||||
|
||||
# Función helper: docker compose up con validación
|
||||
docker_compose_up() {
|
||||
echo "🔐 Validando credenciales..."
|
||||
validate_credentials
|
||||
|
||||
echo "🚀 Iniciando servicios..."
|
||||
docker compose up -d
|
||||
}
|
||||
|
||||
# Exportar alias
|
||||
alias docker_compose_up="docker_compose_up"
|
||||
|
|
@ -1,37 +0,0 @@
|
|||
#!/bin/bash
|
||||
# Validador automático de credenciales para docker compose
|
||||
# Se ejecuta automáticamente antes de iniciar los servicios
|
||||
|
||||
set -e
|
||||
|
||||
# Colores
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
BLUE='\033[0;34m'
|
||||
NC='\033[0m'
|
||||
|
||||
echo -e "${BLUE}🔐 Validando credenciales antes de iniciar...${NC}"
|
||||
|
||||
# Verificar archivo .env
|
||||
if [ ! -f .env ]; then
|
||||
echo -e "${RED}❌ ERROR: Archivo .env no encontrado${NC}"
|
||||
echo -e "${YELLOW}💡 Generando credenciales...${NC}"
|
||||
./generate_secure_credentials.sh --force
|
||||
exit $?
|
||||
fi
|
||||
|
||||
# Verificar credenciales CRÍTICAS
|
||||
for var in POSTGRES_PASSWORD REDIS_PASSWORD DB_PASS; do
|
||||
value=$(grep "^${var}=" .env 2>/dev/null | cut -d'=' -f2 | tr -d ' ')
|
||||
|
||||
if [ -z "$value" ] || [ "$value" = "change_this_to_a_long_random_string" ] || [ "$value" = "change_this_password" ]; then
|
||||
echo -e "${RED}❌ ERROR: $var está vacía o tiene valor por defecto${NC}"
|
||||
echo -e "${YELLOW}💡 Generando credenciales seguras...${NC}"
|
||||
./generate_secure_credentials.sh --force
|
||||
exit $?
|
||||
fi
|
||||
done
|
||||
|
||||
echo -e "${GREEN}✅ Todas las credenciales están definidas${NC}"
|
||||
exit 0
|
||||
|
|
@ -1,13 +0,0 @@
|
|||
.git
|
||||
pgdata
|
||||
pgdata-replica
|
||||
pgdata-replica.old.*
|
||||
pgdata.failed_restore
|
||||
redis-data
|
||||
hf_cache
|
||||
qdrant_storage
|
||||
|
||||
venv
|
||||
__pycache__
|
||||
*.pyc
|
||||
*.log
|
||||
|
|
@ -1,5 +0,0 @@
|
|||
POSTGRES_PASSWORD=test_pass_123456
|
||||
REDIS_PASSWORD=test_redis_123456
|
||||
DB_PASS=test_pass_123456
|
||||
SECRET_KEY=test_secret_key
|
||||
GRAFANA_PASSWORD=test_grafana_123456
|
||||
|
|
@ -1,5 +0,0 @@
|
|||
POSTGRES_PASSWORD=test_pass_123456
|
||||
REDIS_PASSWORD=test_redis_123456
|
||||
DB_PASS=test_pass_123456
|
||||
SECRET_KEY=test_secret_key
|
||||
GRAFANA_PASSWORD=test_grafana_123456
|
||||
|
|
@ -1,92 +0,0 @@
|
|||
# ==================================================================================
|
||||
# CONFIGURACIÓN SEGURA - Generado automáticamente
|
||||
# Fecha: 2026-04-03 23:21:15
|
||||
# ==================================================================================
|
||||
#
|
||||
# IMPORTANTE:
|
||||
# - NO compartas este archivo
|
||||
# - Guarda las credenciales en un gestor de contraseñas
|
||||
# - Añade .env al .gitignore
|
||||
#
|
||||
# ==================================================================================
|
||||
|
||||
# ==================================================================================
|
||||
# DATABASE CONFIGURATION - PostgreSQL
|
||||
# ==================================================================================
|
||||
POSTGRES_DB=rss
|
||||
POSTGRES_USER=rss
|
||||
POSTGRES_PASSWORD=npti6eFCYsAv71LcJKasnpEAhtHgoD4a
|
||||
|
||||
DB_NAME=rss
|
||||
DB_USER=rss
|
||||
DB_PASS=npti6eFCYsAv71LcJKasnpEAhtHgoD4a
|
||||
DB_HOST=db
|
||||
DB_PORT=5432
|
||||
DB_WRITE_HOST=db
|
||||
DB_READ_HOST=db-replica
|
||||
|
||||
# ==================================================================================
|
||||
# REDIS CONFIGURATION - Con autenticación
|
||||
# ==================================================================================
|
||||
REDIS_HOST=redis
|
||||
REDIS_PORT=6379
|
||||
REDIS_PASSWORD=pmBJtSBMiYGptknv5pXa163kUR946V25
|
||||
|
||||
# ==================================================================================
|
||||
# APPLICATION SECRETS
|
||||
# ==================================================================================
|
||||
SECRET_KEY=4a60ebda58f0716b846de1758e63e209e33070d0dbf8ac3c4ecd0c216e6bb4aa
|
||||
|
||||
# ==================================================================================
|
||||
# MONITORING - Grafana
|
||||
# ==================================================================================
|
||||
GRAFANA_PASSWORD=oTkxxwKCNB25r0unvtNi8AQt
|
||||
|
||||
# ==================================================================================
|
||||
# EXTERNAL SERVICES
|
||||
# ==================================================================================
|
||||
ALLTALK_URL=http://host.docker.internal:7851
|
||||
|
||||
# ==================================================================================
|
||||
# AI MODELS & WORKERS
|
||||
# ==================================================================================
|
||||
RSS_MAX_WORKERS=3
|
||||
TARGET_LANGS=es
|
||||
TRANSLATOR_BATCH=128
|
||||
ENQUEUE=300
|
||||
|
||||
# RSS Ingestor Configuration
|
||||
RSS_POKE_INTERVAL_MIN=15
|
||||
RSS_MAX_FAILURES=10
|
||||
RSS_FEED_TIMEOUT=60
|
||||
|
||||
# URL Feed Discovery Worker
|
||||
URL_DISCOVERY_INTERVAL_MIN=15
|
||||
URL_DISCOVERY_BATCH_SIZE=10
|
||||
MAX_FEEDS_PER_URL=5
|
||||
|
||||
# CTranslate2 / AI Model Paths
|
||||
CT2_MODEL_PATH=/app/models/nllb-ct2
|
||||
CT2_DEVICE=cuda
|
||||
CT2_COMPUTE_TYPE=int8_float16
|
||||
UNIVERSAL_MODEL=facebook/nllb-200-distilled-600M
|
||||
|
||||
# Embeddings
|
||||
EMB_MODEL=sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2
|
||||
EMB_BATCH=64
|
||||
EMB_DEVICE=cuda
|
||||
|
||||
# NER
|
||||
NER_LANG=es
|
||||
NER_BATCH=64
|
||||
|
||||
# Flask / Gunicorn
|
||||
GUNICORN_WORKERS=8
|
||||
FLASK_DEBUG=0
|
||||
|
||||
# Qdrant Configuration
|
||||
QDRANT_HOST=qdrant
|
||||
QDRANT_PORT=6333
|
||||
QDRANT_COLLECTION_NAME=news_vectors
|
||||
QDRANT_BATCH_SIZE=100
|
||||
QDRANT_SLEEP_IDLE=30
|
||||
|
|
@ -1,92 +0,0 @@
|
|||
# ==================================================================================
|
||||
# CONFIGURACIÓN SEGURA - Generado automáticamente
|
||||
# Fecha: 2026-04-03 23:21:15
|
||||
# ==================================================================================
|
||||
#
|
||||
# IMPORTANTE:
|
||||
# - NO compartas este archivo
|
||||
# - Guarda las credenciales en un gestor de contraseñas
|
||||
# - Añade .env al .gitignore
|
||||
#
|
||||
# ==================================================================================
|
||||
|
||||
# ==================================================================================
|
||||
# DATABASE CONFIGURATION - PostgreSQL
|
||||
# ==================================================================================
|
||||
POSTGRES_DB=rss
|
||||
POSTGRES_USER=rss
|
||||
POSTGRES_PASSWORD=npti6eFCYsAv71LcJKasnpEAhtHgoD4a
|
||||
|
||||
DB_NAME=rss
|
||||
DB_USER=rss
|
||||
DB_PASS=npti6eFCYsAv71LcJKasnpEAhtHgoD4a
|
||||
DB_HOST=db
|
||||
DB_PORT=5432
|
||||
DB_WRITE_HOST=db
|
||||
DB_READ_HOST=db-replica
|
||||
|
||||
# ==================================================================================
|
||||
# REDIS CONFIGURATION - Con autenticación
|
||||
# ==================================================================================
|
||||
REDIS_HOST=redis
|
||||
REDIS_PORT=6379
|
||||
REDIS_PASSWORD=pmBJtSBMiYGptknv5pXa163kUR946V25
|
||||
|
||||
# ==================================================================================
|
||||
# APPLICATION SECRETS
|
||||
# ==================================================================================
|
||||
SECRET_KEY=4a60ebda58f0716b846de1758e63e209e33070d0dbf8ac3c4ecd0c216e6bb4aa
|
||||
|
||||
# ==================================================================================
|
||||
# MONITORING - Grafana
|
||||
# ==================================================================================
|
||||
GRAFANA_PASSWORD=oTkxxwKCNB25r0unvtNi8AQt
|
||||
|
||||
# ==================================================================================
|
||||
# EXTERNAL SERVICES
|
||||
# ==================================================================================
|
||||
ALLTALK_URL=http://host.docker.internal:7851
|
||||
|
||||
# ==================================================================================
|
||||
# AI MODELS & WORKERS
|
||||
# ==================================================================================
|
||||
RSS_MAX_WORKERS=3
|
||||
TARGET_LANGS=es
|
||||
TRANSLATOR_BATCH=128
|
||||
ENQUEUE=300
|
||||
|
||||
# RSS Ingestor Configuration
|
||||
RSS_POKE_INTERVAL_MIN=15
|
||||
RSS_MAX_FAILURES=10
|
||||
RSS_FEED_TIMEOUT=60
|
||||
|
||||
# URL Feed Discovery Worker
|
||||
URL_DISCOVERY_INTERVAL_MIN=15
|
||||
URL_DISCOVERY_BATCH_SIZE=10
|
||||
MAX_FEEDS_PER_URL=5
|
||||
|
||||
# CTranslate2 / AI Model Paths
|
||||
CT2_MODEL_PATH=/app/models/nllb-ct2
|
||||
CT2_DEVICE=cuda
|
||||
CT2_COMPUTE_TYPE=int8_float16
|
||||
UNIVERSAL_MODEL=facebook/nllb-200-distilled-600M
|
||||
|
||||
# Embeddings
|
||||
EMB_MODEL=sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2
|
||||
EMB_BATCH=64
|
||||
EMB_DEVICE=cuda
|
||||
|
||||
# NER
|
||||
NER_LANG=es
|
||||
NER_BATCH=64
|
||||
|
||||
# Flask / Gunicorn
|
||||
GUNICORN_WORKERS=8
|
||||
FLASK_DEBUG=0
|
||||
|
||||
# Qdrant Configuration
|
||||
QDRANT_HOST=qdrant
|
||||
QDRANT_PORT=6333
|
||||
QDRANT_COLLECTION_NAME=news_vectors
|
||||
QDRANT_BATCH_SIZE=100
|
||||
QDRANT_SLEEP_IDLE=30
|
||||
|
|
@ -1,92 +0,0 @@
|
|||
# ==================================================================================
|
||||
# CONFIGURACIÓN SEGURA - Generado automáticamente
|
||||
# Fecha: 2026-04-03 23:30:04
|
||||
# ==================================================================================
|
||||
#
|
||||
# IMPORTANTE:
|
||||
# - NO compartas este archivo
|
||||
# - Guarda las credenciales en un gestor de contraseñas
|
||||
# - Añade .env al .gitignore
|
||||
#
|
||||
# ==================================================================================
|
||||
|
||||
# ==================================================================================
|
||||
# DATABASE CONFIGURATION - PostgreSQL
|
||||
# ==================================================================================
|
||||
POSTGRES_DB=rss
|
||||
POSTGRES_USER=rss
|
||||
POSTGRES_PASSWORD=SI84OnqRT40FtdAeQro3eSJULKzFJ06F
|
||||
|
||||
DB_NAME=rss
|
||||
DB_USER=rss
|
||||
DB_PASS=SI84OnqRT40FtdAeQro3eSJULKzFJ06F
|
||||
DB_HOST=db
|
||||
DB_PORT=5432
|
||||
DB_WRITE_HOST=db
|
||||
DB_READ_HOST=db-replica
|
||||
|
||||
# ==================================================================================
|
||||
# REDIS CONFIGURATION - Con autenticación
|
||||
# ==================================================================================
|
||||
REDIS_HOST=redis
|
||||
REDIS_PORT=6379
|
||||
REDIS_PASSWORD=MvdWZlO1YCZ3QxE761bbvEd2qx44dO9j
|
||||
|
||||
# ==================================================================================
|
||||
# APPLICATION SECRETS
|
||||
# ==================================================================================
|
||||
SECRET_KEY=68d5b0297d44e88f5ab83720e0bdf8eeef2ceeccd72b8e46b7f1b47531571780
|
||||
|
||||
# ==================================================================================
|
||||
# MONITORING - Grafana
|
||||
# ==================================================================================
|
||||
GRAFANA_PASSWORD=BVWprtQiW2KDmaDR8zZ2Br83
|
||||
|
||||
# ==================================================================================
|
||||
# EXTERNAL SERVICES
|
||||
# ==================================================================================
|
||||
ALLTALK_URL=http://host.docker.internal:7851
|
||||
|
||||
# ==================================================================================
|
||||
# AI MODELS & WORKERS
|
||||
# ==================================================================================
|
||||
RSS_MAX_WORKERS=3
|
||||
TARGET_LANGS=es
|
||||
TRANSLATOR_BATCH=128
|
||||
ENQUEUE=300
|
||||
|
||||
# RSS Ingestor Configuration
|
||||
RSS_POKE_INTERVAL_MIN=15
|
||||
RSS_MAX_FAILURES=10
|
||||
RSS_FEED_TIMEOUT=60
|
||||
|
||||
# URL Feed Discovery Worker
|
||||
URL_DISCOVERY_INTERVAL_MIN=15
|
||||
URL_DISCOVERY_BATCH_SIZE=10
|
||||
MAX_FEEDS_PER_URL=5
|
||||
|
||||
# CTranslate2 / AI Model Paths
|
||||
CT2_MODEL_PATH=/app/models/nllb-ct2
|
||||
CT2_DEVICE=cuda
|
||||
CT2_COMPUTE_TYPE=int8_float16
|
||||
UNIVERSAL_MODEL=facebook/nllb-200-distilled-600M
|
||||
|
||||
# Embeddings
|
||||
EMB_MODEL=sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2
|
||||
EMB_BATCH=64
|
||||
EMB_DEVICE=cuda
|
||||
|
||||
# NER
|
||||
NER_LANG=es
|
||||
NER_BATCH=64
|
||||
|
||||
# Flask / Gunicorn
|
||||
GUNICORN_WORKERS=8
|
||||
FLASK_DEBUG=0
|
||||
|
||||
# Qdrant Configuration
|
||||
QDRANT_HOST=qdrant
|
||||
QDRANT_PORT=6333
|
||||
QDRANT_COLLECTION_NAME=news_vectors
|
||||
QDRANT_BATCH_SIZE=100
|
||||
QDRANT_SLEEP_IDLE=30
|
||||
|
|
@ -1,92 +0,0 @@
|
|||
# ==================================================================================
|
||||
# CONFIGURACIÓN SEGURA - Generado automáticamente
|
||||
# Fecha: 2026-04-03 23:31:40
|
||||
# ==================================================================================
|
||||
#
|
||||
# IMPORTANTE:
|
||||
# - NO compartas este archivo
|
||||
# - Guarda las credenciales en un gestor de contraseñas
|
||||
# - Añade .env al .gitignore
|
||||
#
|
||||
# ==================================================================================
|
||||
|
||||
# ==================================================================================
|
||||
# DATABASE CONFIGURATION - PostgreSQL
|
||||
# ==================================================================================
|
||||
POSTGRES_DB=rss
|
||||
POSTGRES_USER=rss
|
||||
POSTGRES_PASSWORD=S5xwsI9HKjdaBWLpCkqp0mA0DJYZerpD
|
||||
|
||||
DB_NAME=rss
|
||||
DB_USER=rss
|
||||
DB_PASS=S5xwsI9HKjdaBWLpCkqp0mA0DJYZerpD
|
||||
DB_HOST=db
|
||||
DB_PORT=5432
|
||||
DB_WRITE_HOST=db
|
||||
DB_READ_HOST=db-replica
|
||||
|
||||
# ==================================================================================
|
||||
# REDIS CONFIGURATION - Con autenticación
|
||||
# ==================================================================================
|
||||
REDIS_HOST=redis
|
||||
REDIS_PORT=6379
|
||||
REDIS_PASSWORD=cVF79QuJulPO5auEhA2nNqv7mqAruzYh
|
||||
|
||||
# ==================================================================================
|
||||
# APPLICATION SECRETS
|
||||
# ==================================================================================
|
||||
SECRET_KEY=2eb7c221245bb8d896f83255188de614e59fae6b8dfbf16eee8a23c60dde4ffa
|
||||
|
||||
# ==================================================================================
|
||||
# MONITORING - Grafana
|
||||
# ==================================================================================
|
||||
GRAFANA_PASSWORD=R4J61V6OGLTCesrITwEvUPTm
|
||||
|
||||
# ==================================================================================
|
||||
# EXTERNAL SERVICES
|
||||
# ==================================================================================
|
||||
ALLTALK_URL=http://host.docker.internal:7851
|
||||
|
||||
# ==================================================================================
|
||||
# AI MODELS & WORKERS
|
||||
# ==================================================================================
|
||||
RSS_MAX_WORKERS=3
|
||||
TARGET_LANGS=es
|
||||
TRANSLATOR_BATCH=128
|
||||
ENQUEUE=300
|
||||
|
||||
# RSS Ingestor Configuration
|
||||
RSS_POKE_INTERVAL_MIN=15
|
||||
RSS_MAX_FAILURES=10
|
||||
RSS_FEED_TIMEOUT=60
|
||||
|
||||
# URL Feed Discovery Worker
|
||||
URL_DISCOVERY_INTERVAL_MIN=15
|
||||
URL_DISCOVERY_BATCH_SIZE=10
|
||||
MAX_FEEDS_PER_URL=5
|
||||
|
||||
# CTranslate2 / AI Model Paths
|
||||
CT2_MODEL_PATH=/app/models/nllb-ct2
|
||||
CT2_DEVICE=cuda
|
||||
CT2_COMPUTE_TYPE=int8_float16
|
||||
UNIVERSAL_MODEL=facebook/nllb-200-distilled-600M
|
||||
|
||||
# Embeddings
|
||||
EMB_MODEL=sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2
|
||||
EMB_BATCH=64
|
||||
EMB_DEVICE=cuda
|
||||
|
||||
# NER
|
||||
NER_LANG=es
|
||||
NER_BATCH=64
|
||||
|
||||
# Flask / Gunicorn
|
||||
GUNICORN_WORKERS=8
|
||||
FLASK_DEBUG=0
|
||||
|
||||
# Qdrant Configuration
|
||||
QDRANT_HOST=qdrant
|
||||
QDRANT_PORT=6333
|
||||
QDRANT_COLLECTION_NAME=news_vectors
|
||||
QDRANT_BATCH_SIZE=100
|
||||
QDRANT_SLEEP_IDLE=30
|
||||
67
.env.example
67
.env.example
|
|
@ -1,67 +0,0 @@
|
|||
# Database Configuration
|
||||
POSTGRES_DB=rss
|
||||
POSTGRES_USER=rss
|
||||
POSTGRES_PASSWORD=change_this_password
|
||||
DB_NAME=rss
|
||||
DB_USER=rss
|
||||
DB_PASS=change_this_password
|
||||
DB_HOST=db
|
||||
DB_PORT=5432
|
||||
DB_WRITE_HOST=db
|
||||
DB_READ_HOST=db-replica
|
||||
|
||||
# Redis Configuration
|
||||
REDIS_HOST=redis
|
||||
REDIS_PORT=6379
|
||||
|
||||
# Application Secrets
|
||||
SECRET_KEY=change_this_to_a_long_random_string
|
||||
|
||||
# External Services
|
||||
ALLTALK_URL=http://host.docker.internal:7851
|
||||
|
||||
# AI Models & Workers
|
||||
RSS_MAX_WORKERS=3
|
||||
# Translation Pipeline
|
||||
TARGET_LANGS=es
|
||||
TRANSLATOR_BATCH=16
|
||||
SCHEDULER_BATCH=2000
|
||||
SCHEDULER_SLEEP=30
|
||||
LANG_DETECT_BATCH=1000
|
||||
LANG_DETECT_SLEEP=60
|
||||
|
||||
# RSS Ingestor Configuration
|
||||
RSS_POKE_INTERVAL_MIN=15
|
||||
RSS_MAX_FAILURES=10
|
||||
RSS_FEED_TIMEOUT=60
|
||||
|
||||
# URL Feed Discovery Worker
|
||||
URL_DISCOVERY_INTERVAL_MIN=15
|
||||
URL_DISCOVERY_BATCH_SIZE=10
|
||||
MAX_FEEDS_PER_URL=5
|
||||
|
||||
# CTranslate2 / AI Model Paths
|
||||
CT2_MODEL_PATH=/app/models/nllb-ct2
|
||||
CT2_DEVICE=cuda
|
||||
CT2_COMPUTE_TYPE=int8_float16
|
||||
UNIVERSAL_MODEL=facebook/nllb-200-distilled-600M
|
||||
|
||||
# Embeddings
|
||||
EMB_MODEL=sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2
|
||||
EMB_BATCH=64
|
||||
EMB_DEVICE=cuda
|
||||
|
||||
# NER
|
||||
NER_LANG=es
|
||||
NER_BATCH=64
|
||||
|
||||
# Flask / Gunicorn
|
||||
GUNICORN_WORKERS=8
|
||||
FLASK_DEBUG=0
|
||||
|
||||
# Qdrant Configuration
|
||||
QDRANT_HOST=qdrant
|
||||
QDRANT_PORT=6333
|
||||
QDRANT_COLLECTION_NAME=news_vectors
|
||||
QDRANT_BATCH_SIZE=100
|
||||
QDRANT_SLEEP_IDLE=30
|
||||
|
|
@ -1,117 +0,0 @@
|
|||
# ==================================================================================
|
||||
# SEGURIDAD: CONFIGURACIÓN DE PRODUCCIÓN
|
||||
# ==================================================================================
|
||||
#
|
||||
# IMPORTANTE:
|
||||
# 1. Copia este archivo a .env
|
||||
# 2. Cambia TODOS los valores de contraseñas y secrets
|
||||
# 3. NO compartas este archivo en repositorios públicos
|
||||
# 4. Añade .env al .gitignore
|
||||
#
|
||||
# ==================================================================================
|
||||
|
||||
# ==================================================================================
|
||||
# DATABASE CONFIGURATION - PostgreSQL
|
||||
# ==================================================================================
|
||||
POSTGRES_DB=rss
|
||||
POSTGRES_USER=rss
|
||||
# CRÍTICO: Genera una contraseña fuerte (mínimo 32 caracteres aleatorios)
|
||||
# Ejemplo para generar: openssl rand -base64 32
|
||||
POSTGRES_PASSWORD=CAMBIAR_ESTO_POR_UNA_CONTRASEÑA_FUERTE_DE_32_CARACTERES
|
||||
|
||||
DB_NAME=rss
|
||||
DB_USER=rss
|
||||
DB_PASS=CAMBIAR_ESTO_POR_UNA_CONTRASEÑA_FUERTE_DE_32_CARACTERES
|
||||
DB_HOST=db
|
||||
DB_PORT=5432
|
||||
DB_WRITE_HOST=db
|
||||
DB_READ_HOST=db-replica
|
||||
|
||||
# ==================================================================================
|
||||
# REDIS CONFIGURATION - Autenticación habilitada
|
||||
# ==================================================================================
|
||||
REDIS_HOST=redis
|
||||
REDIS_PORT=6379
|
||||
# CRÍTICO: Genera una contraseña fuerte para Redis
|
||||
# Ejemplo: openssl rand -base64 32
|
||||
REDIS_PASSWORD=CAMBIAR_ESTO_POR_UNA_CONTRASEÑA_FUERTE_REDIS
|
||||
|
||||
# ==================================================================================
|
||||
# APPLICATION SECRETS
|
||||
# ==================================================================================
|
||||
# CRÍTICO: Secret key para Flask - debe ser único y secreto
|
||||
# Genera con: python -c "import secrets; print(secrets.token_hex(32))"
|
||||
SECRET_KEY=CAMBIAR_ESTO_POR_UN_TOKEN_HEX_DE_64_CARACTERES
|
||||
|
||||
# ==================================================================================
|
||||
# MONITORING - Grafana
|
||||
# ==================================================================================
|
||||
# IMPORTANTE: Cambia el password de admin de Grafana
|
||||
GRAFANA_PASSWORD=CAMBIAR_ESTO_POR_UNA_CONTRASEÑA_FUERTE_GRAFANA
|
||||
|
||||
# ==================================================================================
|
||||
# EXTERNAL SERVICES
|
||||
# ==================================================================================
|
||||
ALLTALK_URL=http://host.docker.internal:7851
|
||||
|
||||
# ==================================================================================
|
||||
# AI MODELS & WORKERS
|
||||
# ==================================================================================
|
||||
RSS_MAX_WORKERS=3
|
||||
TARGET_LANGS=es
|
||||
TRANSLATOR_BATCH=128
|
||||
ENQUEUE=300
|
||||
|
||||
# RSS Ingestor Configuration
|
||||
RSS_POKE_INTERVAL_MIN=15
|
||||
RSS_MAX_FAILURES=10
|
||||
RSS_FEED_TIMEOUT=60
|
||||
|
||||
# URL Feed Discovery Worker
|
||||
URL_DISCOVERY_INTERVAL_MIN=15
|
||||
URL_DISCOVERY_BATCH_SIZE=10
|
||||
MAX_FEEDS_PER_URL=5
|
||||
|
||||
# CTranslate2 / AI Model Paths
|
||||
CT2_MODEL_PATH=/app/models/nllb-ct2
|
||||
CT2_DEVICE=cuda
|
||||
CT2_COMPUTE_TYPE=int8_float16
|
||||
UNIVERSAL_MODEL=facebook/nllb-200-distilled-600M
|
||||
|
||||
# Embeddings
|
||||
EMB_MODEL=sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2
|
||||
EMB_BATCH=64
|
||||
EMB_DEVICE=cuda
|
||||
|
||||
# NER
|
||||
NER_LANG=es
|
||||
NER_BATCH=64
|
||||
|
||||
# Flask / Gunicorn
|
||||
GUNICORN_WORKERS=8
|
||||
FLASK_DEBUG=0
|
||||
|
||||
# Qdrant Configuration
|
||||
QDRANT_HOST=qdrant
|
||||
QDRANT_PORT=6333
|
||||
QDRANT_COLLECTION_NAME=news_vectors
|
||||
QDRANT_BATCH_SIZE=100
|
||||
QDRANT_SLEEP_IDLE=30
|
||||
|
||||
# ==================================================================================
|
||||
# COMANDOS ÚTILES PARA GENERAR CONTRASEÑAS SEGURAS
|
||||
# ==================================================================================
|
||||
#
|
||||
# PostgreSQL Password (32 caracteres):
|
||||
# openssl rand -base64 32
|
||||
#
|
||||
# Redis Password (32 caracteres):
|
||||
# openssl rand -base64 32
|
||||
#
|
||||
# Flask Secret Key (64 hex chars):
|
||||
# python -c "import secrets; print(secrets.token_hex(32))"
|
||||
#
|
||||
# Grafana Password (fuerte):
|
||||
# openssl rand -base64 24
|
||||
#
|
||||
# ==================================================================================
|
||||
140
.gitignore
vendored
140
.gitignore
vendored
|
|
@ -1,120 +1,80 @@
|
|||
# ==================================================================================
|
||||
# Python
|
||||
# ==================================================================================
|
||||
__pycache__/
|
||||
*.py[cod]
|
||||
*$py.class
|
||||
*.so
|
||||
*.pyc
|
||||
*.pyo
|
||||
*.pyd
|
||||
.Python
|
||||
build/
|
||||
develop-eggs/
|
||||
dist/
|
||||
downloads/
|
||||
eggs/
|
||||
.eggs/
|
||||
lib/
|
||||
lib64/
|
||||
parts/
|
||||
sdist/
|
||||
var/
|
||||
wheels/
|
||||
*.egg-info/
|
||||
.installed.cfg
|
||||
*.so
|
||||
*.egg
|
||||
|
||||
# Virtual environments
|
||||
*.egg-info/
|
||||
dist/
|
||||
build/
|
||||
venv/
|
||||
env/
|
||||
ENV/
|
||||
.venv
|
||||
|
||||
# ==================================================================================
|
||||
# IDEs
|
||||
# ==================================================================================
|
||||
.env
|
||||
.vscode/
|
||||
.idea/
|
||||
*.swp
|
||||
*.swo
|
||||
*~
|
||||
.DS_Store
|
||||
Thumbs.db
|
||||
|
||||
# ==================================================================================
|
||||
# SECURITY - NEVER COMMIT
|
||||
# ==================================================================================
|
||||
.env
|
||||
.env.backup
|
||||
.env.backup2
|
||||
.env.generated
|
||||
.env.local
|
||||
.env.*.local
|
||||
|
||||
# Credentials
|
||||
*.pem
|
||||
*.key
|
||||
*.crt
|
||||
credentials.json
|
||||
service-account.json
|
||||
|
||||
# ==================================================================================
|
||||
# RSS2 Data & Models (downloaded at runtime)
|
||||
# ==================================================================================
|
||||
# ML Models - downloaded/converted by workers
|
||||
models/
|
||||
models/nllb-ct2/
|
||||
models/nllb-ct2-1.3b/
|
||||
|
||||
# Wikipedia images - downloaded by wiki_worker
|
||||
data/wiki_images/
|
||||
data/backups/
|
||||
backups/
|
||||
|
||||
# Database storage (PostgreSQL)
|
||||
data/pgdata/
|
||||
pgdata/
|
||||
pgdata-replica/
|
||||
pgdata.failed_restore/
|
||||
pgdata-replica.old.*/
|
||||
|
||||
# Redis data
|
||||
data/redis-data/
|
||||
redis-data/
|
||||
|
||||
# Qdrant vector storage
|
||||
data/qdrant_storage/
|
||||
qdrant_storage/
|
||||
|
||||
# HuggingFace cache (downloaded models)
|
||||
hf_cache/
|
||||
models/nllb-ct2/
|
||||
qdrant_storage/
|
||||
*.log
|
||||
*.db
|
||||
*.sqlite
|
||||
*.sqlite3
|
||||
data/
|
||||
*.mp4
|
||||
*.mp3
|
||||
*.wav
|
||||
*.srt
|
||||
*.tar.gz
|
||||
*.zip
|
||||
*.bak
|
||||
*.old
|
||||
|
||||
# Monitoring data
|
||||
monitoring/data/
|
||||
|
||||
# ==================================================================================
|
||||
# Backup files
|
||||
# SECURITY FILES - NEVER COMMIT THESE
|
||||
# ==================================================================================
|
||||
# Environment files with credentials
|
||||
.env
|
||||
.env.backup*
|
||||
.env.generated
|
||||
.env.local
|
||||
.env.*.local
|
||||
|
||||
# Database backups
|
||||
*.sql
|
||||
!init-db/*.sql
|
||||
!migrations/*.sql
|
||||
backup_*.sql
|
||||
|
||||
# Redis backups
|
||||
*.rdb
|
||||
redis_backup_*.rdb
|
||||
|
||||
# Qdrant backups
|
||||
qdrant_backup_*.tar.gz
|
||||
|
||||
# ==================================================================================
|
||||
# Temporary files
|
||||
# ==================================================================================
|
||||
tmp/
|
||||
temp/
|
||||
*.tmp
|
||||
*.log
|
||||
|
||||
# ==================================================================================
|
||||
# Docker
|
||||
# ==================================================================================
|
||||
# Docker compose with real credentials (if you create variations)
|
||||
docker-compose.override.yml
|
||||
|
||||
# ==================================================================================
|
||||
# Keep directory structure (optional - uncomment if needed)
|
||||
# models/.gitkeep
|
||||
# data/.gitkeep
|
||||
# Large Language Models
|
||||
models/llm/
|
||||
|
||||
# User Uploads
|
||||
static/uploads/
|
||||
|
||||
# Celery/Workers
|
||||
celerybeat-schedule
|
||||
celerybeat.pid
|
||||
|
||||
# System/IDE
|
||||
.DS_Store
|
||||
Thumbs.db
|
||||
|
|
|
|||
110
AGENT.md
110
AGENT.md
|
|
@ -1,110 +0,0 @@
|
|||
# RSS2 - Agente de Mejora de Software
|
||||
|
||||
## Visión General del Proyecto
|
||||
|
||||
Aplicación de agregación y procesamiento de noticias RSS con capacidades de IA.
|
||||
|
||||
### Stack Tecnológico
|
||||
|
||||
| Capa | Tecnología |
|
||||
|------|-------------|
|
||||
| Frontend | React 18 + TypeScript + Vite + TailwindCSS + React Query + React Router |
|
||||
| Backend API | Go 1.25 + Gin |
|
||||
| Workers Backend | Go 1.25, Python 3.x |
|
||||
| Base de datos | PostgreSQL 18 |
|
||||
| Caché | Redis 7 |
|
||||
| Búsqueda vectorial | Qdrant |
|
||||
| IA/LLM | Ollama, HuggingFace (NLLB, BERT, sentence-transformers) |
|
||||
| Orquestación | Docker Compose |
|
||||
| Monitoreo | Prometheus + Grafana + cAdvisor |
|
||||
|
||||
### Arquitectura de Workers
|
||||
|
||||
- **rss-ingestor-go**: Ingestión de feeds RSS
|
||||
- **scraper**: Extracción de artículos de URLs
|
||||
- **discovery**: Descubrimiento de nuevos feeds
|
||||
- **topics**: Matching de temas y países
|
||||
- **related**: Noticias relacionadas
|
||||
- **langdetect**: Detección de idioma
|
||||
- **translator**: Traducción con NLLB (CPU/GPU)
|
||||
- **embeddings**: Generación de vectores semánticos
|
||||
- **ner**: Extracción de entidades nombradas
|
||||
- **cluster**: Agrupación de noticias
|
||||
- **llm-categorizer**: Categorización con Ollama
|
||||
- **wiki-worker**: Wikipedia info y thumbnails
|
||||
|
||||
---
|
||||
|
||||
## Recomendaciones de Mejora
|
||||
|
||||
### 1. Infraestructura y DevOps
|
||||
|
||||
- **Añadir CI/CD**: Pipeline con GitHub Actions para testing y deploy automático
|
||||
- **Health checks**: Implementar health checks en todos los workers (no solo en db/redis)
|
||||
- **Logging centralizado**: Integrar Loki o ELK stack para agregación de logs
|
||||
- **Variables de entorno**: Mover configuración hardcodeada (ej: `translator_type: cpu`) a variables与环境
|
||||
|
||||
### 2. Backend Go
|
||||
|
||||
- **Estructura**: El proyecto no tiene estructura clara de packages (cmd/ internal/)
|
||||
- **Testing**: No se observan tests unitarios en el backend
|
||||
- **Config**: Usar Viper o library dedicada para configuración
|
||||
- **Graceful shutdown**: Implementar en todos los workers
|
||||
- **Migraciones**: Usar herramientas como golang-migrate en lugar de crear tablas en initDB()
|
||||
|
||||
### 3. Frontend
|
||||
|
||||
- **E2E Testing**: Añadir Playwright o Cypress
|
||||
- **State Management**: Considerar Zustand o Jotai para estado global
|
||||
- **Componentes**: Crear library de componentes reutilizables
|
||||
- **i18n**: Preparar para multiidioma
|
||||
- **PWA**: Añadir Service Worker para offline
|
||||
|
||||
### 4. Base de Datos
|
||||
|
||||
- **ORM/Query Builder**: Considerar GORM o sqlc para mejor mantenimiento
|
||||
- **Índices**: Verificar que todos los campos de búsqueda tengan índices
|
||||
- **Migrations**: Sistema formal de migraciones
|
||||
- **Connection Pooling**: Configurar apropiadamente
|
||||
|
||||
### 5. Workers y Procesamiento
|
||||
|
||||
- **Colas**: Implementar RabbitMQ o Redis Streams para jobs asíncronos
|
||||
- **Retry Logic**: Sistema robusto de reintentos con backoff exponencial
|
||||
- **Dead Letter Queue**: Manejo de trabajos fallidos
|
||||
- **Métricas por worker**: Prometheus metrics específicas por worker
|
||||
- **Rate Limiting**: Protecciones contra rate limits de APIs externas
|
||||
|
||||
### 6. Seguridad
|
||||
|
||||
- **Rate limiting API**: Implementar en endpoints sensibles
|
||||
- **Input Validation**: Usar biblioteca como go-playground/validator
|
||||
- **CSRF Protection**: Añadir tokens CSRF
|
||||
- **Security Headers**: Implementar todos los headers de seguridad
|
||||
- **Audit Logs**: Logging de acciones administrativas
|
||||
|
||||
### 7. Rendimiento
|
||||
|
||||
- **Caching**: Cachear más endpoints con Redis (ej: estadísticas)
|
||||
- **Pagination**: Implementar cursor-based pagination para grandes datasets
|
||||
- **Database Connection Pooling**: Configurar límites apropiados
|
||||
- **Image Optimization**: Comprimir imágenes antes de servir
|
||||
|
||||
### 8. Documentación
|
||||
|
||||
- **API Docs**: OpenAPI/Swagger para la REST API
|
||||
- **Arquitecture Decision Records (ADRs)**: Documentar decisiones técnicas
|
||||
- **Runbooks**: Documentación de operaciones
|
||||
|
||||
### 9. Monitorización y Alertas
|
||||
|
||||
- **Alertas**: Configurar alertas en Grafana para workers caídos
|
||||
- **Dashboards**: Dashboard específico por worker
|
||||
- **Tracing**: Añadir OpenTelemetry para distributed tracing
|
||||
|
||||
### 10. User Experience
|
||||
|
||||
- **Dark Mode**: Soporte completo
|
||||
- **Keyboard Shortcuts**: Mejorar navegación
|
||||
- **Infinite Scroll**: Para feeds grandes
|
||||
- **Real-time Updates**: WebSockets para nuevas noticias
|
||||
408
AGENTS.md
408
AGENTS.md
|
|
@ -1,408 +0,0 @@
|
|||
# AGENTS.md - RSS2 Development Guide
|
||||
|
||||
## 🛠️ Build & Test Commands
|
||||
|
||||
### Backend (Go)
|
||||
```bash
|
||||
cd backend && go mod tidy
|
||||
cd backend && go build -o ../bin/server ./cmd/server
|
||||
cd rss-ingestor-go && go build -o ../bin/rss-ingestor .
|
||||
|
||||
# Single test
|
||||
cd backend && go test ./internal/handlers -v -run TestLogin
|
||||
cd backend && go test ./internal/auth -v -run TestGenerateToken
|
||||
```
|
||||
|
||||
### Frontend
|
||||
```bash
|
||||
cd frontend && npm install
|
||||
cd frontend && npm run dev # Development
|
||||
cd frontend && npm run build # Production build
|
||||
cd frontend && npm test # Run tests
|
||||
cd frontend && npm run test:ui # UI mode
|
||||
```
|
||||
|
||||
### Makefile
|
||||
```bash
|
||||
make build # Build all binaries
|
||||
make clean # Remove binaries
|
||||
make docker-build # Build Docker images
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 📁 Project Structure
|
||||
```
|
||||
rss2/
|
||||
├── backend/ # Go API server + Workers
|
||||
│ ├── cmd/
|
||||
│ │ ├── server/main.go # API REST principal (Gin)
|
||||
│ │ ├── wiki_worker/main.go # Wikipedia integration
|
||||
│ │ ├── qdrant/main.go # Vector indexing worker
|
||||
│ │ ├── related/main.go # Related news worker
|
||||
│ │ ├── topics/main.go # Country/topic matcher
|
||||
│ │ ├── scraper/main.go # Deep scraping worker
|
||||
│ │ ├── discovery/main.go # RSS feed discovery
|
||||
│ │ └── topics/main.go # Topic matching
|
||||
│ └── internal/
|
||||
│ ├── handlers/ # HTTP endpoints
|
||||
│ ├── models/ # Data models
|
||||
│ ├── auth/ # JWT authentication
|
||||
│ ├── middleware/ # CORS, Auth middleware
|
||||
│ ├── services/ # ML services (Translate, Embeddings, NER, Semantic Search)
|
||||
│ ├── db/ # PostgreSQL connection
|
||||
│ ├── cache/ # Redis connection
|
||||
│ └── config/ # Configuration loading
|
||||
├── frontend/ # React + TypeScript + Vite
|
||||
│ └── src/
|
||||
│ ├── pages/ # Home, News, Search, Admin, Feeds, Stats
|
||||
│ ├── components/ # Layout, UI components
|
||||
│ └── services/ # API client
|
||||
├── workers/ # Python workers
|
||||
│ ├── ctranslator_worker.py # NLLB-200 translation (CTranslate2)
|
||||
│ ├── ner_worker.py # Spacy NER + Topic extraction
|
||||
│ ├── embeddings_worker.py # Sentence transformers embeddings
|
||||
│ ├── cluster_worker.py # News clustering
|
||||
│ ├── langdetect_worker.py # Language detection
|
||||
│ ├── llm_categorizer_worker.py # Ollama LLM categorization
|
||||
│ ├── simple_categorizer_worker.py
|
||||
│ ├── simple_translator.py
|
||||
│ ├── simple_translator_worker.py
|
||||
│ ├── translation_worker.py
|
||||
│ ├── translation_scheduler.py
|
||||
│ └── remote_translator_worker.py # Remote GPU worker via WebSocket
|
||||
├── data/ # PostgreSQL, Redis, Qdrant data
|
||||
├── models/ # ML models (nllb-ct2)
|
||||
├── hf_cache/ # HuggingFace cache
|
||||
├── init-db/ # SQL migrations
|
||||
├── monitoring/ # Prometheus + Grafana config
|
||||
├── rss-ingestor-go/ # RSS crawler (Go)
|
||||
└── docker-compose.yml # Full stack orchestration
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 🚀 Servicios del Sistema
|
||||
|
||||
### Capa de Acceso y API (Puerto 8888)
|
||||
| Servicio | Tecnología | Descripción |
|
||||
|---------|------------|-------------|
|
||||
| **nginx** | Nginx Alpine | Gateway y Proxy Inverso |
|
||||
| **rss2_frontend** | React + Vite | Interfaz web responsiva |
|
||||
| **backend-go** | Go + Gin | API REST principal |
|
||||
|
||||
### Ingesta y Descubrimiento (Go)
|
||||
| Servicio | Tecnología | Descripción |
|
||||
|---------|------------|-------------|
|
||||
| **rss-ingestor-go** | Go | Crawler RSS de alto rendimiento |
|
||||
| **scraper** | Go | Scraper profundo con sanitización HTML |
|
||||
| **discovery** | Go | Agente de descubrimiento de feeds RSS |
|
||||
|
||||
### Procesamiento de Datos e IA
|
||||
| Servicio | Tecnología | Descripción |
|
||||
|---------|------------|-------------|
|
||||
| **translator** | NLLB-200 (CPU) | Traducción neuronal CTranslate2 |
|
||||
| **translator-gpu** | NLLB-200 (GPU) | Traducción acelerada CUDA |
|
||||
| **remote-translator** | WebSocket | Worker GPU remoto |
|
||||
| **embeddings** | S-Transformers | Generación de vectores semánticos |
|
||||
| **ner** | Spacy + BERT | Reconocimiento de entidades (PER, ORG, LOC) |
|
||||
| **llm-categorizer** | Ollama/Mistral | Clasificación con modelos de lenguaje |
|
||||
| **wiki-worker** | Go | Integración Wikipedia + thumbnails |
|
||||
| **topics** | Go | Matcher de países y temas |
|
||||
| **related** | Go | Detección de noticias relacionadas |
|
||||
| **qdrant-worker** | Go | Vectorización + búsqueda semántica |
|
||||
| **cluster** | Python | Agrupación de noticias |
|
||||
| **langdetect** | Python | Detección de idioma |
|
||||
| **translation-scheduler** | Python | Creador de tareas de traducción |
|
||||
|
||||
### Capa de Almacenamiento
|
||||
| Servicio | Tecnología | Descripción |
|
||||
|---------|------------|-------------|
|
||||
| **db** | PostgreSQL 18 | Base de datos relacional |
|
||||
| **qdrant** | Qdrant | Base de datos vectorial |
|
||||
| **redis** | Redis 7 | Cache y colas de mensajes |
|
||||
|
||||
### Monitoreo
|
||||
| Servicio | Tecnología | Descripción |
|
||||
|---------|------------|-------------|
|
||||
| **prometheus** | Prometheus | Métricas del sistema |
|
||||
| **grafana** | Grafana | Dashboard (puerto 3001) |
|
||||
| **cadvisor** | cAdvisor | Monitoreo Docker |
|
||||
|
||||
---
|
||||
|
||||
## 📝 Code Style Guidelines
|
||||
|
||||
### Backend (Go)
|
||||
|
||||
#### Imports
|
||||
```go
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/rss2/backend/internal/auth"
|
||||
"github.com/rss2/backend/internal/models"
|
||||
)
|
||||
```
|
||||
**Order:** Standard → Third-party → Local packages
|
||||
**Use:** `goimports` to auto-format
|
||||
|
||||
#### Naming
|
||||
- Packages: lowercase (`handlers`, `services`)
|
||||
- Functions/Methods: camelCase (`GetNews`, `CreateUser`)
|
||||
- Variables: camelCase (`userId`, `newsList`)
|
||||
- Constants: UPPER_SNAKE_CASE (`MaxPageSize`)
|
||||
- Types: PascalCase (`NewsResponse`, `User`)
|
||||
|
||||
#### Error Handling
|
||||
```go
|
||||
func CreateResource(c *gin.Context) {
|
||||
var req CreateRequest
|
||||
if err := c.ShouldBindJSON(&req); err != nil {
|
||||
c.JSON(http.StatusBadRequest, models.ErrorResponse{
|
||||
Error: "Invalid request",
|
||||
Message: err.Error(),
|
||||
})
|
||||
return
|
||||
}
|
||||
// ...
|
||||
}
|
||||
```
|
||||
**Always return errors** | **Use `models.ErrorResponse`**
|
||||
|
||||
#### Database
|
||||
```go
|
||||
err := db.GetPool().QueryRow(ctx, "SELECT * FROM users WHERE id = $1", id).Scan(&user)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusNotFound, models.ErrorResponse{Error: "User not found"})
|
||||
return
|
||||
}
|
||||
```
|
||||
**Use context** | **Named parameters ($1, $2)**
|
||||
|
||||
#### Tests
|
||||
```go
|
||||
func TestLoginInvalidRequest(t *testing.T) {
|
||||
router := gin.New()
|
||||
router.POST("/auth/login", Login)
|
||||
body := []byte(`{}`)
|
||||
req, _ := http.NewRequest("POST", "/auth/login", bytes.NewBuffer(body))
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
w := httptest.NewRecorder()
|
||||
router.ServeHTTP(w, req)
|
||||
if w.Code != http.StatusBadRequest {
|
||||
t.Errorf("expected 400, got %d", w.Code)
|
||||
}
|
||||
}
|
||||
```
|
||||
**Use `gin.TestMode`** | **Test error paths**
|
||||
|
||||
---
|
||||
|
||||
### Frontend (TypeScript + React)
|
||||
|
||||
#### Imports
|
||||
```tsx
|
||||
import React, { useState, useEffect } from 'react'
|
||||
import { Routes, Route } from 'react-router-dom'
|
||||
import { Layout } from './components/layout/Layout'
|
||||
import { api } from './services/api'
|
||||
```
|
||||
**Order:** React → Router → Components → Services → Utils
|
||||
|
||||
#### Components
|
||||
```tsx
|
||||
function NewsList() {
|
||||
const [news, setNews] = useState<News[]>([])
|
||||
const [loading, setLoading] = useState(true)
|
||||
const [error, setError] = useState<string | null>(null)
|
||||
|
||||
useEffect(() => { fetchNews(); }, [])
|
||||
|
||||
const fetchNews = async () => {
|
||||
try {
|
||||
const res = await api.get('/news')
|
||||
setNews(res.data)
|
||||
} catch (err) {
|
||||
setError(err.message)
|
||||
}
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="news-list">
|
||||
{loading && <Spinner />}
|
||||
{error && <ErrorBanner message={error} />}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
```
|
||||
|
||||
#### Types
|
||||
```tsx
|
||||
interface News {
|
||||
id: number
|
||||
title: string
|
||||
summary: string
|
||||
url: string
|
||||
publishedAt: string
|
||||
}
|
||||
```
|
||||
|
||||
#### Styling
|
||||
- Use Tailwind CSS utility classes
|
||||
- Avoid inline styles
|
||||
- Use `clsx` for conditional classes
|
||||
|
||||
#### TypeScript Rules
|
||||
- Use `strict: true` mode
|
||||
- Avoid `any` - define interfaces
|
||||
- Use type guards for narrowing
|
||||
- Prefer optional chaining `?.`
|
||||
|
||||
---
|
||||
|
||||
## 🔒 Security Guidelines
|
||||
|
||||
1. **Never commit secrets** - Use `.env` in `.gitignore`
|
||||
2. **Validate all inputs** - Use `go-playground/validator`
|
||||
3. **Prepared statements** - Prevent SQL injection
|
||||
4. **Rate limiting** - On sensitive endpoints
|
||||
5. **HTTPS only** - Enforce in production
|
||||
|
||||
### Variables de Entorno Críticas
|
||||
```bash
|
||||
POSTGRES_PASSWORD # Contraseña PostgreSQL
|
||||
REDIS_PASSWORD # Contraseña Redis
|
||||
DB_PASS # Contraseña para workers
|
||||
SECRET_KEY # Key JWT
|
||||
GRAFANA_PASSWORD # Dashboard password
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 🧪 Testing Best Practices
|
||||
|
||||
### Backend
|
||||
- Test error cases and edge cases
|
||||
- Use `gin.TestMode` for HTTP tests
|
||||
|
||||
### Frontend
|
||||
- Test component rendering
|
||||
- Test API integration
|
||||
- Test error states
|
||||
- Use vitest
|
||||
|
||||
---
|
||||
|
||||
## 📦 Deployment
|
||||
|
||||
```bash
|
||||
# Generar credenciales seguras
|
||||
./pre-deploy.sh --generate
|
||||
|
||||
# Validar y desplegar
|
||||
./pre-deploy.sh
|
||||
|
||||
# O manualmente
|
||||
docker compose up -d
|
||||
|
||||
# Escalar workers de traducción
|
||||
docker compose up -d --scale translator-gpu=4
|
||||
```
|
||||
|
||||
### Escalado de Workers GPU
|
||||
```bash
|
||||
# 1 worker GPU (8GB+ VRAM)
|
||||
docker compose up -d --scale translator-gpu=1
|
||||
|
||||
# 2 workers GPU (16GB+ VRAM)
|
||||
docker compose up -d --scale translator-gpu=2
|
||||
|
||||
# 4 workers GPU (32GB+ VRAM)
|
||||
docker compose up -d --scale translator-gpu=4
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 🔧 Configuración de Workers
|
||||
|
||||
### Environment Variables Principales
|
||||
|
||||
| Variable | Descripción | Default |
|
||||
|----------|-------------|---------|
|
||||
| `DB_HOST` | Host PostgreSQL | localhost |
|
||||
| `DB_PORT` | Puerto PostgreSQL | 5432 |
|
||||
| `DB_NAME` | Nombre base de datos | rss |
|
||||
| `DB_USER` | Usuario PostgreSQL | rss |
|
||||
| `DB_PASS` | Contraseña PostgreSQL | - |
|
||||
| `TARGET_LANGS` | Idiomas destino | es |
|
||||
| `TRANSLATOR_BATCH` | Tamaño de batch | 32 |
|
||||
| `CT2_DEVICE` | Dispositivo (cpu/cuda) | cpu |
|
||||
| `CT2_COMPUTE_TYPE` | Tipo (int8/float16) | int8 |
|
||||
| `NER_BATCH` | Batch NER | 64 |
|
||||
| `EMB_BATCH` | Batch embeddings | 64 |
|
||||
|
||||
---
|
||||
|
||||
## 📊 Endpoints de API Principales
|
||||
|
||||
### News
|
||||
- `GET /api/news` - Listar noticias (paginado, filtros)
|
||||
- `GET /api/news/:id` - Ver noticia con entidades
|
||||
- `DELETE /api/news/:id` - Eliminar noticia (admin)
|
||||
|
||||
### Feeds
|
||||
- `GET /api/feeds` - Listar feeds
|
||||
- `POST /api/feeds` - Crear feed (auth)
|
||||
- `PUT /api/feeds/:id` - Actualizar feed (auth)
|
||||
- `DELETE /api/feeds/:id` - Eliminar feed (auth)
|
||||
|
||||
### Search
|
||||
- `GET /api/search?q=...` - Búsqueda texto
|
||||
- `GET /api/search?q=...&semantic=true` - Búsqueda semántica
|
||||
|
||||
### Entities
|
||||
- `GET /api/entities?tipo=persona` - Listar entidades (PER, ORG, LOC)
|
||||
|
||||
### Admin
|
||||
- `GET /api/admin/backup` - Backup SQL completo
|
||||
- `GET /api/admin/backup/news` - Backup noticias (ZIP)
|
||||
- `GET /api/admin/users` - Listar usuarios
|
||||
- `POST /api/admin/workers/start` - Iniciar workers traducción
|
||||
- `POST /api/admin/workers/stop` - Detener workers traducción
|
||||
- `GET /api/admin/workers/status` - Estado de workers
|
||||
|
||||
### Auth
|
||||
- `POST /api/auth/login` - Iniciar sesión
|
||||
- `POST /api/auth/register` - Registrarse
|
||||
- `GET /api/auth/me` - Usuario actual (auth)
|
||||
|
||||
### Stats
|
||||
- `GET /api/stats` - Estadísticas globales
|
||||
|
||||
---
|
||||
|
||||
## 🎯 Capabilities del Sistema
|
||||
|
||||
1. **Enriquecimiento Wikipedia**: Detecta personas/orgs, descarga biografías e imágenes
|
||||
2. **Categorización LLM**: Clasificación con Mistral-7B vía Ollama
|
||||
3. **Búsqueda Semántica**: Qdrant vector search con mxbai-embed-large
|
||||
4. **Traducción Neuronal**: NLLB-200 (600M-1.3B params) CPU/GPU
|
||||
5. **NER**: Spacy es_core_news_lg para entidades nombradas
|
||||
6. **Noticias Relacionadas**: Similitud coseno entre embeddings
|
||||
7. **Detección de Idioma**: langdetect
|
||||
8. **Clustering**: Agrupación automática de noticias
|
||||
9. **WebSocket Workers**: Workers GPU remotos conectados por WS
|
||||
10. **Backup Automático**: pg_dump con compresión ZIP
|
||||
|
||||
---
|
||||
|
||||
## 📖 Documentación Adicional
|
||||
|
||||
- [README.md](../README.md) - Guía de despliegue completo
|
||||
- [DEPLOY.md](./DEPLOY.md) - Instrucciones de producción
|
||||
- [SECURITY_GUIDE.md](./SECURITY_GUIDE.md) - Guía de seguridad
|
||||
- [QUICKSTART_LLM.md](./QUICKSTART_LLM.md) - Configuración LLM
|
||||
- [remote-worker.md](./remote-worker.md) - Workers remotos
|
||||
353
DEPLOY.md
353
DEPLOY.md
|
|
@ -1,353 +0,0 @@
|
|||
# 🚀 Guía de Despliegue RSS2
|
||||
|
||||
## ⚡ Despliegue Rápido (5 minutos)
|
||||
|
||||
### Paso 1: Clonar el Proyecto
|
||||
|
||||
```bash
|
||||
git clone https://github.com/tu-usuario/rss2.git
|
||||
cd rss2
|
||||
```
|
||||
|
||||
### Paso 2: Generar Credenciales Seguras
|
||||
|
||||
```bash
|
||||
./pre-deploy.sh --generate
|
||||
```
|
||||
|
||||
**Salida esperada:**
|
||||
```
|
||||
🔑 Generando credenciales seguras...
|
||||
|
||||
⚠️ IMPORTANTE: Guarda estas credenciales en un lugar seguro
|
||||
|
||||
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
|
||||
POSTGRES_PASSWORD: S5xwsI9HKjdaBWLpCkqp0mA0DJYZerpD
|
||||
REDIS_PASSWORD: cVF79QuJulPO5auEhA2nNqv7mqAruzYh
|
||||
SECRET_KEY: 2eb7c221245bb8d896f83255188de614e59fae6b8dfbf16eee8a23c60dde4ffa
|
||||
GRAFANA_PASSWORD: R4J61V6OGLTCesrITwEvUPTm
|
||||
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
|
||||
```
|
||||
|
||||
**⚠️ IMPORTANTE:** Copia estas contraseñas y guárdalas en un gestor de contraseñas. ¡Nunca se volverán a mostrar!
|
||||
|
||||
### Paso 3: Validar y Desplegar
|
||||
|
||||
```bash
|
||||
./pre-deploy.sh
|
||||
```
|
||||
|
||||
**El script te preguntará:**
|
||||
```
|
||||
¿Deseas desplegar ahora?
|
||||
1) Desplegar automáticamente (recomendado)
|
||||
2) Solo validar y salir (modo manual)
|
||||
Elige una opción:
|
||||
```
|
||||
|
||||
**Opción A: Despliegue automático (Opción 1)**
|
||||
- Elige `1` para que el script haga todo automáticamente
|
||||
- Espera a que termine la construcción de las imágenes
|
||||
- Espera a que inicien los servicios
|
||||
|
||||
**Opción B: Despliegue manual (Opción 2)**
|
||||
- Elige `2` para validar solo
|
||||
- Luego ejecuta manualmente:
|
||||
```bash
|
||||
docker compose up -d
|
||||
```
|
||||
|
||||
### Paso 4: Verificar Despliegue
|
||||
|
||||
```bash
|
||||
# Verificar estado de los servicios
|
||||
docker compose ps
|
||||
|
||||
# Deberías ver algo como:
|
||||
# Name Status
|
||||
# rss2_db Up (healthy)
|
||||
# rss2_redis Up (healthy)
|
||||
# rss2_backend_go Up (running)
|
||||
# ...
|
||||
```
|
||||
|
||||
### Paso 5: Acceder a la Aplicación
|
||||
|
||||
```bash
|
||||
# Aplicación web
|
||||
http://localhost:8888
|
||||
|
||||
# API (prueba con curl)
|
||||
curl http://localhost:8888/api/feeds
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 🔐 Seguridad
|
||||
|
||||
### ¿Por qué es importante?
|
||||
|
||||
**Sin seguridad (❌ MAL):**
|
||||
```bash
|
||||
docker compose up -d
|
||||
WARN: The "DB_PASS" variable is not set. Defaulting to a blank string.
|
||||
# Las contraseñas están vacías ¡PELIGRO!
|
||||
```
|
||||
|
||||
**Con seguridad (✅ BIEN):**
|
||||
```bash
|
||||
./pre-deploy.sh --generate
|
||||
./pre-deploy.sh
|
||||
docker compose up -d
|
||||
# ✅ Todo funciona correctamente
|
||||
```
|
||||
|
||||
### Variables Críticas
|
||||
|
||||
| Variable | Qué es | Por qué es importante |
|
||||
|----------|--------|----------------------|
|
||||
| `POSTGRES_PASSWORD` | Contraseña de PostgreSQL | Base de datos principal |
|
||||
| `REDIS_PASSWORD` | Contraseña de Redis | Cache y colas |
|
||||
| `DB_PASS` | Contraseña para workers | Conexiones de workers |
|
||||
| `SECRET_KEY` | Key secreta de la app | JWT, encriptación |
|
||||
| `GRAFANA_PASSWORD` | Contraseña de Grafana | Dashboard de monitoring |
|
||||
|
||||
### Guardar tus Credenciales
|
||||
|
||||
**Opción 1: Gestor de contraseñas (Recomendado)**
|
||||
```bash
|
||||
# Copia las credenciales y guárdalas en:
|
||||
# - LastPass
|
||||
# - 1Password
|
||||
# - KeePass
|
||||
# - Bitwarden
|
||||
```
|
||||
|
||||
**Opción 2: Archivo seguro local**
|
||||
```bash
|
||||
# Crea un archivo seguro fuera del proyecto
|
||||
echo "POSTGRES_PASSWORD=..." > ~/rss2-credentials.txt
|
||||
echo "REDIS_PASSWORD=..." >> ~/rss2-credentials.txt
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 🛠️ Comandos Útiles
|
||||
|
||||
### Verificar Credenciales
|
||||
```bash
|
||||
# Ver que las contraseñas están definidas
|
||||
grep -E "^(POSTGRES_PASSWORD|REDIS_PASSWORD|DB_PASS)=" .env
|
||||
```
|
||||
|
||||
### Reiniciar Servicios
|
||||
```bash
|
||||
# Reiniciar todos los servicios
|
||||
docker compose restart
|
||||
|
||||
# Reiniciar solo base de datos
|
||||
docker compose restart db
|
||||
```
|
||||
|
||||
### Escalar Workers
|
||||
```bash
|
||||
# Añadir más traductores (CPU)
|
||||
docker compose up -d --scale translator=3
|
||||
|
||||
# Añadir más traductores (GPU)
|
||||
docker compose up -d --scale translator-gpu=2
|
||||
```
|
||||
|
||||
### Ver Logs
|
||||
```bash
|
||||
# Ver logs en tiempo real
|
||||
docker compose logs -f
|
||||
|
||||
# Ver logs específicos
|
||||
docker compose logs -f db
|
||||
docker compose logs -f redis
|
||||
docker compose logs -f backend-go
|
||||
```
|
||||
|
||||
### Limpiar y Reiniciar
|
||||
```bash
|
||||
# Detener y eliminar todo (incluye datos)
|
||||
docker compose down -v
|
||||
|
||||
# Eliminar solo contenedores (mantiene datos)
|
||||
docker compose down
|
||||
|
||||
# Eliminar volumen específico
|
||||
docker volume rm rss2_db
|
||||
docker volume rm rss2_redis
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 🐛 Solución de Problemas
|
||||
|
||||
### Problema: WARN sobre variables no definidas
|
||||
|
||||
```
|
||||
WARN: The "DB_PASS" variable is not set. Defaulting to a blank string.
|
||||
```
|
||||
|
||||
**Solución:**
|
||||
```bash
|
||||
# Ejecutar pre-deploy para generar credenciales
|
||||
./pre-deploy.sh --generate
|
||||
```
|
||||
|
||||
### Problema: Contraseña vacía en healthcheck
|
||||
|
||||
```
|
||||
redis-cli: Authentication failed
|
||||
```
|
||||
|
||||
**Solución:**
|
||||
```bash
|
||||
# Verificar REDIS_PASSWORD
|
||||
grep REDIS_PASSWORD .env
|
||||
|
||||
# Si está vacío, regenerar
|
||||
./pre-deploy.sh --generate
|
||||
```
|
||||
|
||||
### Problema: Port 8888 en uso
|
||||
|
||||
```
|
||||
ERROR: failed to start service: port 8888 already in use
|
||||
```
|
||||
|
||||
**Solución:**
|
||||
```bash
|
||||
# Cambiar puerto en docker-compose.yml
|
||||
# O matar el proceso que usa el puerto
|
||||
lsof -ti:8888 | xargs kill
|
||||
|
||||
# O usar otro puerto
|
||||
docker compose up -d --scale nginx=1
|
||||
```
|
||||
|
||||
### Problema: Base de datos no inicia
|
||||
|
||||
```
|
||||
postgres: cannot connect to database
|
||||
```
|
||||
|
||||
**Solución:**
|
||||
```bash
|
||||
# Ver logs de la base de datos
|
||||
docker compose logs db
|
||||
|
||||
# Reiniciar base de datos
|
||||
docker compose restart db
|
||||
|
||||
# Si persiste, eliminar y recrear (¡PERDERÁS DATOS!)
|
||||
docker compose down -v
|
||||
docker compose up -d db
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 📊 Estado de los Servicios
|
||||
|
||||
### Servicios Críticos
|
||||
|
||||
| Servicio | Puerto | Descripción |
|
||||
|----------|--------|-------------|
|
||||
| `nginx` | 8888 | Aplicación web |
|
||||
| `backend-go` | 8080 | API interna |
|
||||
| `db` | 5432 | PostgreSQL |
|
||||
| `redis` | 6379 | Redis cache |
|
||||
| `qdrant` | 6333 | Vector database |
|
||||
|
||||
### Dashboard de Monitorización
|
||||
|
||||
| Herramienta | Puerto | Acceso |
|
||||
|-------------|--------|--------|
|
||||
| Grafana | 3001 | http://127.0.0.1:3001 |
|
||||
| Prometheus | 9090 | http://127.0.0.1:9090 |
|
||||
|
||||
**Nota:** Accede a Grafana desde `127.0.0.1` (localhost) por seguridad.
|
||||
|
||||
---
|
||||
|
||||
## 🔄 Actualización (Upgrade)
|
||||
|
||||
### Actualizar a Nueva Versión
|
||||
|
||||
```bash
|
||||
# 1. Pull las nuevas imágenes
|
||||
docker compose pull
|
||||
|
||||
# 2. Backup de la base de datos
|
||||
docker compose exec db pg_dump -U rss rss > backup.sql
|
||||
|
||||
# 3. Parar servicios
|
||||
docker compose down
|
||||
|
||||
# 4. Actualizar código
|
||||
git pull
|
||||
|
||||
# 5. Reiniciar
|
||||
docker compose up -d
|
||||
|
||||
# 6. Verificar
|
||||
curl http://localhost:8888/api/health
|
||||
```
|
||||
|
||||
### Migrar Contraseñas
|
||||
|
||||
Si tienes credenciales existentes y quieres mantenerlas:
|
||||
```bash
|
||||
# 1. Verificar .env existente
|
||||
grep POSTGRES_PASSWORD .env
|
||||
|
||||
# 2. Si quieres nuevas credenciales
|
||||
./pre-deploy.sh --generate
|
||||
|
||||
# 3. Si quieres mantener las existentes
|
||||
# No hagas nada, las credenciales seguirán funcionando
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 📝 Archivos Importantes
|
||||
|
||||
| Archivo | Propósito |
|
||||
|---------|-----------|
|
||||
| `.env` | Contraseñas y configuración (NO subir a Git) |
|
||||
| `.env.example` | Plantilla (SÍ subir a Git) |
|
||||
| `pre-deploy.sh` | Script de validación (SÍ subir a Git) |
|
||||
| `generate_secure_credentials.sh` | Script de generación (SÍ subir a Git) |
|
||||
| `docker-compose.yml` | Configuración de Docker (SÍ subir a Git) |
|
||||
|
||||
---
|
||||
|
||||
## ✅ Checklist Final
|
||||
|
||||
Antes de considerar tu despliegue como "listo", verifica:
|
||||
|
||||
- [ ] `.env` tiene credenciales generadas (no por defecto)
|
||||
- [ ] `.env` está en `.gitignore`
|
||||
- [ ] Puedes acceder a http://localhost:8888
|
||||
- [ ] `docker compose ps` muestra todos los servicios "Up"
|
||||
- [ ] La base de datos muestra "healthy"
|
||||
- [ ] Los logs de la base de datos no tienen errores
|
||||
- [ ] Redis responde a ping
|
||||
- [ ] Qdrant está funcionando
|
||||
|
||||
**¡Listo! Tu RSS2 está desplegado y funcionando.** 🎉
|
||||
|
||||
---
|
||||
|
||||
## 📚 Documentación Adicional
|
||||
|
||||
- [README.md](./README.md) - Documentación general
|
||||
- [SECURITY_GUIDE.md](./SECURITY_GUIDE.md) - Guía de seguridad
|
||||
- [DOCKER.md](./DOCKER.md) - Configuración avanzada de Docker
|
||||
|
||||
---
|
||||
|
||||
**RSS2** - *Transformando noticias en inteligencia con IA.*
|
||||
50
Dockerfile
50
Dockerfile
|
|
@ -1,50 +0,0 @@
|
|||
FROM python:3.11-slim
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
libpq-dev gcc git curl \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
ENV PYTHONUNBUFFERED=1 \
|
||||
PIP_DISABLE_PIP_VERSION_CHECK=1 \
|
||||
TOKENIZERS_PARALLELISM=false \
|
||||
HF_HOME=/root/.cache/huggingface
|
||||
|
||||
COPY requirements.txt .
|
||||
RUN pip install --no-cache-dir --upgrade pip
|
||||
|
||||
RUN pip install --no-cache-dir torch==2.1.0 torchvision==0.16.0 --index-url https://download.pytorch.org/whl/cu121
|
||||
|
||||
RUN pip install --no-cache-dir \
|
||||
ctranslate2 \
|
||||
sentencepiece \
|
||||
transformers==4.44.0 \
|
||||
protobuf==3.20.3 \
|
||||
"numpy<2" \
|
||||
psycopg2-binary \
|
||||
redis \
|
||||
requests \
|
||||
beautifulsoup4 \
|
||||
lxml \
|
||||
langdetect \
|
||||
nltk \
|
||||
scikit-learn \
|
||||
pandas \
|
||||
sentence-transformers \
|
||||
spacy
|
||||
|
||||
RUN python -m spacy download es_core_news_lg
|
||||
|
||||
COPY workers/ ./workers/
|
||||
COPY init-db/ ./init-db/
|
||||
COPY migrations/ ./migrations/
|
||||
COPY entity_config.json .
|
||||
|
||||
ENV DB_HOST=db
|
||||
ENV DB_PORT=5432
|
||||
ENV DB_NAME=rss
|
||||
ENV DB_USER=rss
|
||||
ENV DB_PASS=x
|
||||
|
||||
CMD ["python", "-m", "workers.embeddings_worker"]
|
||||
|
|
@ -1,35 +0,0 @@
|
|||
FROM golang:1.22-alpine AS builder
|
||||
|
||||
ENV GOTOOLCHAIN=auto
|
||||
|
||||
RUN apk add --no-cache git
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY backend/go.mod ./
|
||||
RUN go mod download
|
||||
|
||||
COPY backend/ ./
|
||||
|
||||
RUN go mod tidy
|
||||
|
||||
COPY backend/ ./
|
||||
|
||||
RUN CGO_ENABLED=0 GOOS=linux go build -o /bin/discovery ./cmd/discovery
|
||||
|
||||
FROM alpine:3.19
|
||||
|
||||
RUN apk add --no-cache ca-certificates tzdata
|
||||
|
||||
COPY --from=builder /bin/discovery /bin/discovery
|
||||
|
||||
ENV DB_HOST=db \
|
||||
DB_PORT=5432 \
|
||||
DB_NAME=rss \
|
||||
DB_USER=rss \
|
||||
DB_PASS=rss \
|
||||
DISCOVERY_INTERVAL=900 \
|
||||
DISCOVERY_BATCH=10 \
|
||||
MAX_FEEDS_PER_URL=5
|
||||
|
||||
ENTRYPOINT ["/bin/discovery"]
|
||||
|
|
@ -1,38 +0,0 @@
|
|||
FROM golang:1.22-alpine AS builder
|
||||
|
||||
ENV GOTOOLCHAIN=auto
|
||||
|
||||
RUN apk add --no-cache git
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY backend/go.mod ./
|
||||
RUN go mod download
|
||||
|
||||
COPY backend/ ./
|
||||
|
||||
RUN go mod tidy
|
||||
|
||||
COPY backend/ ./
|
||||
|
||||
RUN CGO_ENABLED=0 GOOS=linux go build -o /bin/qdrant-worker ./cmd/qdrant
|
||||
|
||||
FROM alpine:3.19
|
||||
|
||||
RUN apk add --no-cache ca-certificates tzdata
|
||||
|
||||
COPY --from=builder /bin/qdrant-worker /bin/qdrant-worker
|
||||
|
||||
ENV DB_HOST=db \
|
||||
DB_PORT=5432 \
|
||||
DB_NAME=rss \
|
||||
DB_USER=rss \
|
||||
DB_PASS=rss \
|
||||
QDRANT_HOST=qdrant \
|
||||
QDRANT_PORT=6333 \
|
||||
QDRANT_COLLECTION=news_vectors \
|
||||
OLLAMA_URL=http://ollama:11434 \
|
||||
QDRANT_SLEEP=30 \
|
||||
QDRANT_BATCH=100
|
||||
|
||||
ENTRYPOINT ["/bin/qdrant-worker"]
|
||||
|
|
@ -1,36 +0,0 @@
|
|||
FROM golang:1.22-alpine AS builder
|
||||
|
||||
ENV GOTOOLCHAIN=auto
|
||||
|
||||
RUN apk add --no-cache git
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY backend/go.mod ./
|
||||
RUN go mod download
|
||||
|
||||
COPY backend/ ./
|
||||
|
||||
RUN go mod tidy
|
||||
|
||||
COPY backend/ ./
|
||||
|
||||
RUN CGO_ENABLED=0 GOOS=linux go build -o /bin/related ./cmd/related
|
||||
|
||||
FROM alpine:3.19
|
||||
|
||||
RUN apk add --no-cache ca-certificates tzdata
|
||||
|
||||
COPY --from=builder /bin/related /bin/related
|
||||
|
||||
ENV DB_HOST=db \
|
||||
DB_PORT=5432 \
|
||||
DB_NAME=rss \
|
||||
DB_USER=rss \
|
||||
DB_PASS=rss \
|
||||
RELATED_SLEEP=10 \
|
||||
RELATED_BATCH=200 \
|
||||
RELATED_TOPK=10 \
|
||||
EMB_MODEL=mxbai-embed-large
|
||||
|
||||
ENTRYPOINT ["/bin/related"]
|
||||
|
|
@ -1,39 +0,0 @@
|
|||
FROM python:3.11-slim-bookworm
|
||||
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
patchelf libpq-dev gcc git curl wget \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
ENV PYTHONUNBUFFERED=1 \
|
||||
PIP_DISABLE_PIP_VERSION_CHECK=1 \
|
||||
TOKENIZERS_PARALLELISM=false \
|
||||
HF_HOME=/root/.cache/huggingface
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY requirements.txt .
|
||||
RUN pip install --no-cache-dir --upgrade pip
|
||||
|
||||
RUN pip install --no-cache-dir torch==2.1.0 torchvision==0.16.0 --index-url https://download.pytorch.org/whl/cu118
|
||||
|
||||
RUN pip install --no-cache-dir \
|
||||
ctranslate2==3.24.0 \
|
||||
sentencepiece \
|
||||
transformers==4.36.0 \
|
||||
protobuf==3.20.3 \
|
||||
"numpy<2" \
|
||||
psycopg2-binary \
|
||||
langdetect \
|
||||
websocket-client
|
||||
|
||||
RUN find /usr/local/lib/python3.11/site-packages/ctranslate2* \
|
||||
-name "libctranslate2-*.so.*" -o -name "libctranslate2.so*" | \
|
||||
xargs -I {} patchelf --clear-execstack {} || true
|
||||
|
||||
COPY workers/remote_translator_worker.py /app/worker.py
|
||||
|
||||
ENV CT2_DEVICE=cuda
|
||||
ENV CT2_COMPUTE_TYPE=float16
|
||||
ENV UNIVERSAL_MODEL=facebook/nllb-200-1.3B
|
||||
|
||||
CMD ["python", "/app/worker.py"]
|
||||
|
|
@ -1,23 +0,0 @@
|
|||
FROM python:3.11-slim
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
libpq-dev \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
ENV PYTHONUNBUFFERED=1
|
||||
|
||||
COPY requirements.txt .
|
||||
RUN pip install --no-cache-dir --upgrade pip
|
||||
RUN pip install --no-cache-dir psycopg2-binary langdetect
|
||||
|
||||
COPY workers/translation_scheduler.py ./workers/
|
||||
|
||||
ENV DB_HOST=db
|
||||
ENV DB_PORT=5432
|
||||
ENV DB_NAME=rss
|
||||
ENV DB_USER=rss
|
||||
ENV DB_PASS=x
|
||||
|
||||
CMD ["python", "workers/translation_scheduler.py"]
|
||||
|
|
@ -1,37 +0,0 @@
|
|||
FROM golang:1.22-alpine AS builder
|
||||
|
||||
ENV GOTOOLCHAIN=auto
|
||||
|
||||
RUN apk add --no-cache git
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY backend/go.mod ./
|
||||
RUN go mod download
|
||||
|
||||
COPY backend/ ./
|
||||
|
||||
RUN go mod tidy
|
||||
|
||||
COPY backend/ ./
|
||||
|
||||
RUN go mod tidy
|
||||
|
||||
RUN CGO_ENABLED=0 GOOS=linux go build -o /bin/scraper ./cmd/scraper
|
||||
|
||||
FROM alpine:3.19
|
||||
|
||||
RUN apk add --no-cache ca-certificates tzdata
|
||||
|
||||
COPY --from=builder /bin/scraper /bin/scraper
|
||||
|
||||
ENV DB_HOST=db \
|
||||
DB_PORT=5432 \
|
||||
DB_NAME=rss \
|
||||
DB_USER=rss \
|
||||
DB_PASS=rss \
|
||||
SCRAPER_SLEEP=60 \
|
||||
SCRAPER_BATCH=10 \
|
||||
SCRAPER_ENRICH_LIMIT=20
|
||||
|
||||
ENTRYPOINT ["/bin/scraper"]
|
||||
|
|
@ -1,34 +0,0 @@
|
|||
FROM golang:1.22-alpine AS builder
|
||||
|
||||
ENV GOTOOLCHAIN=auto
|
||||
|
||||
RUN apk add --no-cache git
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY backend/go.mod ./
|
||||
RUN go mod download
|
||||
|
||||
COPY backend/ ./
|
||||
|
||||
RUN go mod tidy
|
||||
|
||||
COPY backend/ ./
|
||||
|
||||
RUN CGO_ENABLED=0 GOOS=linux go build -o /bin/topics ./cmd/topics
|
||||
|
||||
FROM alpine:3.19
|
||||
|
||||
RUN apk add --no-cache ca-certificates tzdata
|
||||
|
||||
COPY --from=builder /bin/topics /bin/topics
|
||||
|
||||
ENV DB_HOST=db \
|
||||
DB_PORT=5432 \
|
||||
DB_NAME=rss \
|
||||
DB_USER=rss \
|
||||
DB_PASS=rss \
|
||||
TOPICS_SLEEP=10 \
|
||||
TOPICS_BATCH=500
|
||||
|
||||
ENTRYPOINT ["/bin/topics"]
|
||||
|
|
@ -1,44 +0,0 @@
|
|||
FROM python:3.11-slim-bookworm
|
||||
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
patchelf libpq-dev gcc git curl wget \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
ENV PYTHONUNBUFFERED=1 \
|
||||
PIP_DISABLE_PIP_VERSION_CHECK=1 \
|
||||
TOKENIZERS_PARALLELISM=false \
|
||||
HF_HOME=/root/.cache/huggingface
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY requirements.txt .
|
||||
RUN pip install --no-cache-dir --upgrade pip
|
||||
|
||||
RUN pip install --no-cache-dir torch==2.1.0 torchvision==0.16.0 --index-url https://download.pytorch.org/whl/cpu
|
||||
|
||||
RUN pip install --no-cache-dir \
|
||||
ctranslate2==3.24.0 \
|
||||
sentencepiece \
|
||||
transformers==4.36.0 \
|
||||
protobuf==3.20.3 \
|
||||
"numpy<2" \
|
||||
psycopg2-binary \
|
||||
langdetect \
|
||||
redis
|
||||
|
||||
# === ARREGLAR EL EXECUTABLE STACK ===
|
||||
RUN find /usr/local/lib/python3.11/site-packages/ctranslate2* \
|
||||
-name "libctranslate2-*.so.*" -o -name "libctranslate2.so*" | \
|
||||
xargs -I {} patchelf --clear-execstack {} || true
|
||||
|
||||
COPY workers/ ./workers/
|
||||
COPY init-db/ ./init-db/
|
||||
COPY migrations/ ./migrations/
|
||||
|
||||
ENV DB_HOST=db
|
||||
ENV DB_PORT=5432
|
||||
ENV DB_NAME=rss
|
||||
ENV DB_USER=rss
|
||||
ENV DB_PASS=x
|
||||
|
||||
CMD ["python", "-m", "workers.ctranslator_worker"]
|
||||
|
|
@ -1,48 +0,0 @@
|
|||
FROM python:3.11-slim-bookworm
|
||||
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
patchelf libpq-dev gcc git curl wget \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
ENV PYTHONUNBUFFERED=1 \
|
||||
PIP_DISABLE_PIP_VERSION_CHECK=1 \
|
||||
TOKENIZERS_PARALLELISM=false \
|
||||
HF_HOME=/root/.cache/huggingface
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY requirements.txt .
|
||||
RUN pip install --no-cache-dir --upgrade pip
|
||||
|
||||
# Install PyTorch with CUDA support (cu118 for broader compatibility)
|
||||
RUN pip install --no-cache-dir torch==2.1.0 torchvision==0.16.0 --index-url https://download.pytorch.org/whl/cu118
|
||||
|
||||
RUN pip install --no-cache-dir \
|
||||
ctranslate2==3.24.0 \
|
||||
sentencepiece \
|
||||
transformers==4.36.0 \
|
||||
protobuf==3.20.3 \
|
||||
"numpy<2" \
|
||||
psycopg2-binary \
|
||||
langdetect
|
||||
|
||||
# Fix executable stack
|
||||
RUN find /usr/local/lib/python3.11/site-packages/ctranslate2* \
|
||||
-name "libctranslate2-*.so.*" -o -name "libctranslate2.so*" | \
|
||||
xargs -I {} patchelf --clear-execstack {} || true
|
||||
|
||||
COPY workers/ ./workers/
|
||||
COPY init-db/ ./init-db/
|
||||
COPY migrations/ ./migrations/
|
||||
|
||||
ENV DB_HOST=db
|
||||
ENV DB_PORT=5432
|
||||
ENV DB_NAME=rss
|
||||
ENV DB_USER=rss
|
||||
ENV DB_PASS=x
|
||||
|
||||
# GPU Configuration - Override with: docker run --gpus all
|
||||
ENV CT2_DEVICE=cuda
|
||||
ENV CT2_COMPUTE_TYPE=float16
|
||||
|
||||
CMD ["python", "-m", "workers.ctranslator_worker"]
|
||||
|
|
@ -1,35 +0,0 @@
|
|||
FROM golang:alpine AS builder
|
||||
|
||||
ENV GOTOOLCHAIN=auto
|
||||
|
||||
RUN apk add --no-cache git
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY backend/go.mod ./
|
||||
RUN go mod download
|
||||
|
||||
COPY backend/ ./
|
||||
|
||||
RUN go mod tidy
|
||||
|
||||
COPY backend/ ./
|
||||
|
||||
RUN go mod tidy
|
||||
|
||||
RUN CGO_ENABLED=0 GOOS=linux go build -o /bin/wiki_worker ./cmd/wiki_worker
|
||||
|
||||
FROM alpine:3.19
|
||||
|
||||
RUN apk add --no-cache ca-certificates tzdata
|
||||
|
||||
COPY --from=builder /bin/wiki_worker /bin/wiki_worker
|
||||
|
||||
ENV DB_HOST=db \
|
||||
DB_PORT=5432 \
|
||||
DB_NAME=rss \
|
||||
DB_USER=rss \
|
||||
DB_PASS=rss \
|
||||
WIKI_SLEEP=10
|
||||
|
||||
ENTRYPOINT ["/bin/wiki_worker"]
|
||||
33
Makefile
33
Makefile
|
|
@ -1,6 +1,6 @@
|
|||
# RSS2 Workers Makefile
|
||||
|
||||
.PHONY: all build clean deps ingestor scraper discovery topics related qdrant server
|
||||
.PHONY: all build clean deps ingestor scraper discovery topics related qdrant server install rebuild check poc
|
||||
|
||||
# Binary output directory
|
||||
BIN_DIR := bin
|
||||
|
|
@ -69,21 +69,16 @@ run-qdrant:
|
|||
DB_HOST=localhost DB_PORT=5432 DB_NAME=rss DB_USER=rss DB_PASS=rss \
|
||||
QDRANT_HOST=localhost QDRANT_PORT=6333 OLLAMA_URL=http://localhost:11434 $(QDRANT)
|
||||
|
||||
# Docker builds
|
||||
docker-build:
|
||||
docker build -t rss2-ingestor -f rss-ingestor-go/Dockerfile ./rss-ingestor-go
|
||||
docker build -t rss2-server -f backend/Dockerfile ./backend
|
||||
docker build -t rss2-scraper -f Dockerfile.scraper ./backend
|
||||
docker build -t rss2-discovery -f Dockerfile.discovery ./backend
|
||||
docker build -t rss2-topics -f Dockerfile.topics ./backend
|
||||
docker build -t rss2-related -f Dockerfile.related ./backend
|
||||
docker build -t rss2-qdrant -f Dockerfile.qdrant ./backend
|
||||
docker build -t rss2-langdetect -f Dockerfile .
|
||||
docker build -t rss2-scheduler -f Dockerfile.scheduler .
|
||||
docker build -t rss2-translator -f Dockerfile.translator .
|
||||
docker build -t rss2-translator-gpu -f Dockerfile.translator-gpu .
|
||||
docker build -t rss2-embeddings -f Dockerfile.embeddings_worker .
|
||||
docker build -t rss2-ner -f Dockerfile .
|
||||
docker build -t rss2-llm-categorizer -f Dockerfile.llm_worker .
|
||||
docker build -t rss2-frontend -f frontend/Dockerfile ./frontend
|
||||
docker build -t rss2-nginx -f Dockerfile.nginx .
|
||||
# Despliegue en Debian (sin Docker)
|
||||
install:
|
||||
sudo bash deploy/debian/prerequisites.sh
|
||||
sudo bash deploy/debian/install.sh
|
||||
|
||||
rebuild:
|
||||
sudo bash deploy/debian/build.sh
|
||||
|
||||
check:
|
||||
bash deploy/debian/check.sh
|
||||
|
||||
poc:
|
||||
bash poc/poc.sh
|
||||
|
|
|
|||
663
README.md
663
README.md
|
|
@ -1,549 +1,202 @@
|
|||
# RSS2 - AI-Powered News Intelligence Platform
|
||||
# COCONEWS
|
||||
|
||||
RSS2 es una plataforma avanzada de agregación, traducción, análisis y vectorización de noticias, diseñada para transformar flujos masivos de información en inteligencia accionable. Utiliza una arquitectura híbrida de microservicios (Go + Python) integrada con modelos de inteligencia artificial de última generación para ofrecer búsqueda semántica, clasificación inteligente y automatización de contenidos.
|
||||
Plataforma de inteligencia de noticias. Agrega feeds RSS de cualquier idioma, los traduce automáticamente al español, extrae entidades, los agrupa por eventos y los hace buscables semánticamente.
|
||||
|
||||
---
|
||||
|
||||
## 🚀 Capacidades Principales
|
||||
## Qué hace
|
||||
|
||||
* **Enriquecimiento con Wikipedia**: Sistema automatizado que detecta personas y organizaciones, descarga sus biografías e imágenes oficiales de Wikipedia para mostrarlas en tooltips interactivos con avatares circulares.
|
||||
* **Categorización Inteligente (LLM)**: Clasificación de noticias mediante una instancia local de Mistral-7B / Llama-3 (vía Ollama), procesando contenido en tiempo real.
|
||||
* **Búsqueda Semántica**: Motor vectorial Qdrant para descubrir noticias por contexto y significado, yendo más allá de las palabras clave tradicionales.
|
||||
* **Traducción Neuronal de Alta Calidad**: Integración de NLLB-200 (vía CTranslate2) para traducir noticias de múltiples idiomas al español con precisión profesional.
|
||||
* **Inteligencia de Entidades (NER)**: Extracción y normalización automática de Personas, Organizaciones y Lugares para análisis de tendencias y mapeo de relaciones.
|
||||
* **Búsqueda de Noticias Relacionadas**: Algoritmos de similitud que agrupan noticias sobre el mismo tema automáticamente.
|
||||
* **Detección de Idioma**: Identificación automática del idioma de origen para routing correcto a traducción.
|
||||
* **Clustering de Noticias**: Agrupación automática de noticias relacionadas en eventos mediante embeddings.
|
||||
* **Remote Workers GPU**: Workers de traducción GPUremotos conectados vía WebSocket para procesamiento distribuido.
|
||||
* **Backup Automatizado**: Exportación completa de la base de datos en SQL o ZIP.
|
||||
- **Ingesta** feeds RSS/Atom de cualquier idioma de forma continua
|
||||
- **Traduce** al español con NLLB-200 (200 idiomas soportados)
|
||||
- **Extrae entidades** (personas, organizaciones, lugares) con spaCy y las enriquece con Wikipedia
|
||||
- **Genera embeddings** para búsqueda por significado, no solo por palabras clave
|
||||
- **Agrupa noticias** del mismo evento automáticamente
|
||||
- **Categoriza** contenido con reglas y modelos de lenguaje
|
||||
- Interfaz web React con búsqueda semántica, filtros y tooltips de Wikipedia
|
||||
|
||||
---
|
||||
|
||||
## 🏗️ Arquitectura de Servicios (Docker)
|
||||
## Arquitectura
|
||||
|
||||
El sistema se orquestra mediante Docker Compose y se divide en capas especializadas:
|
||||
```
|
||||
Internet (RSS/Atom)
|
||||
│
|
||||
▼
|
||||
rss-ingestor-go ──→ PostgreSQL ──→ langdetect
|
||||
│ │ │
|
||||
scraper/discovery │ translator (NLLB-200)
|
||||
│ │
|
||||
│ embeddings (MiniLM)
|
||||
│ │
|
||||
│ ner (spaCy)
|
||||
│ │
|
||||
│ cluster / related
|
||||
│ │
|
||||
│ qdrant-worker ──→ Qdrant
|
||||
│
|
||||
backend (API REST :8080)
|
||||
│
|
||||
nginx (:8001)
|
||||
│
|
||||
Frontend React
|
||||
```
|
||||
|
||||
### Capa de Acceso y API
|
||||
| Servicio | Tecnología | Descripción |
|
||||
|---------|------------|-------------|
|
||||
| **`nginx`** | Nginx Alpine | Gateway y Proxy Inverso (Puerto **8888**). |
|
||||
| **`rss2_frontend`** | React + Vite | Interfaz web de usuario moderna y responsiva. |
|
||||
| **`backend-go`** | Go + Gin | API REST principal y gestión de lógica de negocio. |
|
||||
|
||||
### Ingesta y Descubrimiento (Go)
|
||||
| Servicio | Tecnología | Descripción |
|
||||
|---------|------------|-------------|
|
||||
| **`rss-ingestor-go`** | Go | Crawler de alto rendimiento para feeds RSS (100 workers). |
|
||||
| **`scraper`** | Go | Scraper profundo con sanitización de HTML y extracción de texto. |
|
||||
| **`discovery`** | Go | Agente autónomo para descubrir nuevos feeds a partir de URLs. |
|
||||
|
||||
### Procesamiento de Datos e IA (Go & Python)
|
||||
| Servicio | Tecnología | Descripción |
|
||||
|---------|------------|-------------|
|
||||
| **`translator`** | NLLB-200 (CPU) | Traducción neuronal optimizada con CTranslate2. |
|
||||
| **`translator-gpu`** | NLLB-200 (GPU) | Traducción acelerada por hardware (CUDA). |
|
||||
| **`remote-translator`** | WebSocket | Worker GPU remoto (conexión vía ws://backend-go:8080/ws/worker). |
|
||||
| **`wiki-worker`** | Go | Integración con Wikipedia y gestión de imágenes locales. |
|
||||
| **`embeddings`** | S-Transformers | Generación de vectores para búsqueda semántica (paraphrase-multilingual-MiniLM-L12-v2). |
|
||||
| **`ner`** | Spacy / es_core_news_lg | Reconocimiento de entidades nombradas (NER). |
|
||||
| **`llm-categorizer`** | Ollama / Mistral | Clasificación avanzada mediante modelos de lenguaje. |
|
||||
| **`topics`** | Go | Matcher automático de países y temas predefinidos. |
|
||||
| **`related`** | Go | Motor de detección de noticias relacionadas (similitud coseno). |
|
||||
| **`qdrant-worker`** | Go | Vectorización y búsqueda semántica con Qdrant + Ollama. |
|
||||
| **`cluster`** | Python | Agrupación de noticias por eventos. |
|
||||
| **`langdetect`** | Python | Detección de idioma para routing de traducciones. |
|
||||
| **`translation-scheduler`** | Python | Creador de tareas de traducción. |
|
||||
|
||||
### Capa de Almacenamiento
|
||||
| Servicio | Tecnología | Descripción |
|
||||
|---------|------------|-------------|
|
||||
| **`db`** | PostgreSQL 18 | Base de datos relacional principal. |
|
||||
| **`qdrant`** | Qdrant | Base de datos vectorial para búsqueda por similitud. |
|
||||
| **`redis`** | Redis 7 | Colas de mensajes y caché de alto desempeño. |
|
||||
|
||||
### Monitoreo
|
||||
| Servicio | Tecnología | Descripción |
|
||||
|---------|------------|-------------|
|
||||
| **`prometheus`** | Prometheus | Métricas del sistema. |
|
||||
| **`grafana`** | Grafana | Dashboard (puerto **3001**). |
|
||||
| **`cadvisor`** | cAdvisor | Monitoreo Docker. |
|
||||
**Stack:**
|
||||
- Go 1.25 — API REST (Gin), ingestor RSS, scraper, workers
|
||||
- Python 3 — Workers ML (NLLB-200, MiniLM, spaCy, CTranslate2)
|
||||
- PostgreSQL 16 — datos relacionales + full-text search
|
||||
- Qdrant — búsqueda vectorial semántica
|
||||
- Redis 7 — caché de consultas
|
||||
- React 18 + TypeScript + Tailwind — frontend
|
||||
- nginx — proxy inverso + archivos estáticos
|
||||
|
||||
---
|
||||
|
||||
## 📊 Endpoints de API
|
||||
## Inicio rápido (POC local)
|
||||
|
||||
### News
|
||||
- `GET /api/news` - Listar noticias (paginado, filtros: q, category_id, country_id, translated_only)
|
||||
- `GET /api/news/:id` - Ver noticia con entidades (incluye Wikipedia enrichment)
|
||||
- `DELETE /api/news/:id` - Eliminar noticia (admin)
|
||||
Prueba COCONEWS en tu máquina en ~2 minutos con datos de muestra, sin instalar los modelos ML.
|
||||
|
||||
### Feeds
|
||||
- `GET /api/feeds` - Listar feeds
|
||||
- `POST /api/feeds` - Crear feed (auth requerido)
|
||||
- `PUT /api/feeds/:id` - Actualizar feed (auth requerido)
|
||||
- `DELETE /api/feeds/:id` - Eliminar feed (auth requerido)
|
||||
- `POST /api/feeds/:id/toggle` - Activar/desactivar feed
|
||||
- `POST /api/feeds/:id/reactivate` - Reactivar feed
|
||||
|
||||
### Search
|
||||
- `GET /api/search?q=...` - Búsqueda texto (filtros: lang, categoria_id, pais_id)
|
||||
- `GET /api/search?q=...&semantic=true` - Búsqueda semántica con Qdrant
|
||||
|
||||
### Entities
|
||||
- `GET /api/entities?tipo=persona|organizacion|lugar` - Listar entidades (con aliasing)
|
||||
|
||||
### Admin
|
||||
- `GET /api/admin/backup` - Backup SQL completo (descargable)
|
||||
- `GET /api/admin/backup/news` - Backup noticias (ZIP)
|
||||
- `GET /api/admin/users` - Listar usuarios
|
||||
- `POST /api/admin/users/:id/promote` - Promover a admin
|
||||
- `POST /api/admin/users/:id/demote` - Quitar admin
|
||||
- `POST /api/admin/reset-db` - Resetear base de datos
|
||||
- `POST /api/admin/aliases` - Crear alias de entidad
|
||||
- `GET /api/admin/aliases/export` - Exportar aliases (CSV)
|
||||
- `POST /api/admin/aliases/import` - Importar aliases (CSV)
|
||||
- `POST /api/admin/entities/retype` - Cambiar tipo de entidad
|
||||
- `POST /api/admin/workers/config` - Configurar workers (type, workers)
|
||||
- `POST /api/admin/workers/start` - Iniciar workers traducción
|
||||
- `POST /api/admin/workers/stop` - Detener workers traducción
|
||||
- `GET /api/admin/workers/status` - Estado de workers
|
||||
- `GET /api/admin/workers/remote` - Listar remote workers
|
||||
- `POST /api/admin/workers/remote` - Crear remote worker
|
||||
- `DELETE /api/admin/workers/remote/:id` - Eliminar remote worker
|
||||
|
||||
### Auth
|
||||
- `POST /api/auth/login` - Iniciar sesión
|
||||
- `POST /api/auth/register` - Registrarse
|
||||
- `GET /api/auth/me` - Usuario actual (auth requerido)
|
||||
- `GET /api/auth/check-first-user` - Verificar si existe primer usuario
|
||||
|
||||
### Stats
|
||||
- `GET /api/stats` - Estadísticas globales (total news, feeds, users, hoy/semana/mes)
|
||||
- `GET /api/categories` - Listar categorías
|
||||
- `GET /api/countries` - Listar países con continentes
|
||||
|
||||
### WebSocket
|
||||
- `GET /ws/worker` - Conexión de remote workers (protocolo de autenticación con API key)
|
||||
|
||||
---
|
||||
|
||||
## 🗄️ Esquema de Base de Datos (Tablas Principales)
|
||||
|
||||
- **noticias**: Artículos RSS (titulo, resumen, url, feed_id, categoria_id, pais_id, lang)
|
||||
- **feeds**: Fuentes RSS (nombre, url, activo, ultimo_fetch)
|
||||
- **traducciones**: Traducciones (noticia_id, lang_from, lang_to, titulo_trad, resumen_trad, status, vectorized)
|
||||
- **tags**: Entidades (valor, tipo: persona/organizacion/lugar/tema, wiki_summary, wiki_url, image_path)
|
||||
- **tags_noticia**: Relación noticia-tag (traduccion_id, noticia_id, tag_id)
|
||||
- **entity_aliases**: Alias de entidades (canonical_name, alias, tipo)
|
||||
- **categorias**: Categorías de noticias
|
||||
- **paises**: Países con continentes
|
||||
- **related_noticias**: Noticias relacionadas (traduccion_id, related_traduccion_id, score)
|
||||
- **traduccion_embeddings**: Embeddings en BD (traduccion_id, model, embedding)
|
||||
- **users**: Usuarios (email, username, password_hash, is_admin)
|
||||
- **config**: Configuración del sistema (translator_type, translator_workers, translator_status)
|
||||
- **remote_workers**: Workers remotos (name, api_key, capabilities, status, last_seen)
|
||||
- **favoritos**: Noticias favoritas de usuarios
|
||||
- **search_history**: Historial de búsquedas
|
||||
|
||||
---
|
||||
|
||||
## ⚙️ Guía de Configuración
|
||||
|
||||
### 1. Requisitos de Hardware
|
||||
* **Modo Básico (CPU)**: 4+ Cores CPU, 8GB RAM.
|
||||
* **Modo Avanzado (IA)**: NVIDIA GPU con 8GB+ VRAM (mínimo recomendado para LLM y Traducción GPU).
|
||||
|
||||
### 2. Instalación Rápida con Seguridad Automática
|
||||
|
||||
#### Opción A: Despliegue con Validación Automática (RECOMENDADO)
|
||||
**Requisitos mínimos para el POC:**
|
||||
- Go 1.25+
|
||||
- Node.js 18+
|
||||
- PostgreSQL (corriendo)
|
||||
- Redis (corriendo)
|
||||
|
||||
```bash
|
||||
# Paso 1: Clonar el proyecto
|
||||
git clone <repo_url>
|
||||
cd rss2
|
||||
git clone https://gitea.laenre.net/pietre/rss2.git coconews
|
||||
cd coconews
|
||||
git checkout coconews
|
||||
|
||||
# Paso 2: Generar credenciales seguras automáticamente
|
||||
./pre-deploy.sh --generate
|
||||
|
||||
# Paso 3: Validar y desplegar
|
||||
./pre-deploy.sh
|
||||
|
||||
# Paso 4: Si eliges opción 2 en el prompt, despliega manualmente
|
||||
docker compose up -d
|
||||
bash poc/poc.sh
|
||||
```
|
||||
|
||||
**Ventajas:**
|
||||
- ✅ Genera credenciales seguras de 32 caracteres
|
||||
- ✅ Valida que no haya contraseñas por defecto
|
||||
- ✅ Muestra resumen de credenciales en consola
|
||||
- ✅ Automático y seguro para producción
|
||||
Abre `http://127.0.0.1:18001` en el navegador.
|
||||
Login: **admin** / **admin123** (creado automáticamente si tienes `pip3 install bcrypt`).
|
||||
|
||||
#### Opción B: Despliegue Manual (Para desarrolladores)
|
||||
Guía detallada para explorar la demo: [docs/POC_GUIDE.md](docs/POC_GUIDE.md)
|
||||
|
||||
---
|
||||
|
||||
## Instalación en servidor Debian
|
||||
|
||||
Para un despliegue completo con todos los workers ML en producción:
|
||||
|
||||
### 1. Instalar prerequisites
|
||||
|
||||
```bash
|
||||
git clone <repo_url>
|
||||
cd rss2
|
||||
sudo bash deploy/debian/prerequisites.sh
|
||||
```
|
||||
|
||||
# Generar credenciales manualmente
|
||||
./generate_secure_credentials.sh
|
||||
Instala: PostgreSQL 16, Redis, nginx, Go 1.25, Node.js 20, Qdrant, Python 3 venv.
|
||||
Pregunta si instalar los modelos ML pesados ahora o después.
|
||||
|
||||
# Copiar credenciales a .env
|
||||
cp .env.generated .env
|
||||
### 2. Configurar entorno
|
||||
|
||||
# Desplegar
|
||||
docker compose up -d
|
||||
```bash
|
||||
cp deploy/debian/env.example /opt/rss2/.env
|
||||
nano /opt/rss2/.env # edita contraseñas y SECRET_KEY
|
||||
```
|
||||
|
||||
### 3. Instalar y arrancar
|
||||
|
||||
```bash
|
||||
sudo bash deploy/debian/install.sh
|
||||
```
|
||||
|
||||
Compila los binarios Go, el frontend React, crea los servicios systemd y arranca todo.
|
||||
|
||||
### Acceder
|
||||
|
||||
```
|
||||
http://IP_DEL_SERVIDOR:8001
|
||||
```
|
||||
|
||||
Guía completa: [docs/DEPLOY_DEBIAN.md](docs/DEPLOY_DEBIAN.md)
|
||||
|
||||
---
|
||||
|
||||
## Gestión de servicios
|
||||
|
||||
```bash
|
||||
# Estado general
|
||||
systemctl status rss2-backend rss2-ingestor rss2-translator
|
||||
|
||||
# Logs en tiempo real
|
||||
journalctl -u rss2-backend -f
|
||||
journalctl -u rss2-translator -f
|
||||
|
||||
# Reiniciar tras actualizar código
|
||||
git pull
|
||||
sudo bash deploy/debian/build.sh
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 🔐 Despliegue con Seguridad Automática (RECOMENDADO)
|
||||
|
||||
### 🛡️ Sistema de Validación Pre-Despliegue
|
||||
|
||||
RSS2 incluye `pre-deploy.sh`, un script inteligente que se ejecuta **antes** de `docker compose` para:
|
||||
|
||||
1. ✅ **Validar credenciales**: Asegura que `POSTGRES_PASSWORD`, `REDIS_PASSWORD`, `DB_PASS` estén definidas
|
||||
2. ✅ **Generar automáticamente**: Crea credenciales seguras de 32 caracteres si faltan
|
||||
3. ✅ **Prevenir errores**: Evita que los WARN de Docker por variables vacías
|
||||
4. ✅ **Mostrar resumen**: Te muestra todas las credenciales en consola
|
||||
|
||||
### 🚀 Despliegue Rápido y Seguro (3 Comandos)
|
||||
## Actualizar el código
|
||||
|
||||
```bash
|
||||
# 1. Generar credenciales seguras automáticamente
|
||||
./pre-deploy.sh --generate
|
||||
|
||||
# 2. Validar y desplegar (elige opción 2 para despliegue manual)
|
||||
./pre-deploy.sh
|
||||
|
||||
# 3. Alternativa: Despliegue manual después de validar
|
||||
./pre-deploy.sh # ← Elige "2) Solo validar y salir"
|
||||
docker compose up -d
|
||||
cd /ruta/al/repo
|
||||
git pull
|
||||
sudo bash deploy/debian/build.sh
|
||||
```
|
||||
|
||||
**Flujo recomendado:**
|
||||
```bash
|
||||
./pre-deploy.sh --generate # Genera .env con credenciales seguras
|
||||
./pre-deploy.sh # Valida y te pregunta si quieres desplegar
|
||||
# Si eliges "2): Solo validar", luego:
|
||||
docker compose up -d # Despliega manualmente
|
||||
```
|
||||
|
||||
### 📋 Opciones del Script `pre-deploy.sh`
|
||||
|
||||
```bash
|
||||
# Validar credenciales existentes
|
||||
./pre-deploy.sh
|
||||
|
||||
# Generar credenciales seguras automáticamente
|
||||
./pre-deploy.sh --generate
|
||||
|
||||
# Saltar validación (NO RECOMENDADO para producción)
|
||||
./pre-deploy.sh --skip
|
||||
|
||||
# Ver ayuda
|
||||
./pre-deploy.sh --help
|
||||
```
|
||||
|
||||
**Salida típica del script:**
|
||||
```
|
||||
╔═══════════════════════════════════════════════════════════╗
|
||||
║ 🔐 Pre-Deploy Security Check - RSS2 Platform ║
|
||||
╚═══════════════════════════════════════════════════════════╝
|
||||
|
||||
✅ Archivo .env encontrado
|
||||
✅ DEFINIDO: POSTGRES_PASSWORD=8x7f2k9m4p1q3w5e
|
||||
✅ DEFINIDO: REDIS_PASSWORD=a9b8c7d6e5f4g3h2
|
||||
✅ DEFINIDO: DB_PASS=8x7f2k9m4p1q3w5e
|
||||
|
||||
📋 Resumen de Credenciales Activas
|
||||
POSTGRES_PASSWORD: ✓
|
||||
REDIS_PASSWORD: ✓
|
||||
DB_PASS: ✓
|
||||
|
||||
✅ VALIDACIÓN COMPLETADA - LISTO PARA DESPLEGAR
|
||||
```
|
||||
|
||||
### 🔒 Variables Críticas que Requieren Contraseñas
|
||||
|
||||
| Variable | Descripción | Uso |
|
||||
|----------|-------------|-----|
|
||||
| `POSTGRES_PASSWORD` | Contraseña de PostgreSQL | Base de datos |
|
||||
| `REDIS_PASSWORD` | Contraseña de Redis | Cache y colas |
|
||||
| `DB_PASS` | Contraseña para workers | Conexiones de workers |
|
||||
| `SECRET_KEY` | Key secreta de la aplicación | JWT, encriptación |
|
||||
| `GRAFANA_PASSWORD` | Contraseña de Grafana | Dashboard de monitoring |
|
||||
`build.sh` recompila los binarios Go, el frontend y sincroniza los workers Python, y reinicia los servicios automáticamente.
|
||||
|
||||
---
|
||||
|
||||
### 3. Escalado de Workers GPU (¡Importante!)
|
||||
## Requisitos de hardware
|
||||
|
||||
#### 🚀 Workers de Traducción GPU Multi-Instancia
|
||||
|
||||
RSS2 soporta múltiples workers de traducción ejecutándose en paralelo utilizando GPUs, lo que acelera significativamente el proceso de traducción masiva.
|
||||
|
||||
**Arquitectura Multi-Worker:**
|
||||
- Cada worker es una instancia independiente del servicio `ctranslator_worker`
|
||||
- Los workers compiten automáticamente por las traducciones pendientes usando `SELECT ... FOR UPDATE SKIP LOCKED`
|
||||
- Cada worker procesa su propio lote (batch) de traducciones sin bloquearse mutuamente
|
||||
- Soporta hasta 8 workers simultáneos (configurable vía panel admin)
|
||||
|
||||
**Requisitos GPU:**
|
||||
```bash
|
||||
# Verificar GPU disponibles
|
||||
nvidia-smi
|
||||
|
||||
# Instalar drivers NVIDIA (si no los tienes)
|
||||
# Ubuntu/Debian:
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y nvidia-driver-535
|
||||
```
|
||||
|
||||
**Despliegue con Múltiples Workers GPU:**
|
||||
|
||||
```bash
|
||||
# Opción 1: Desplegar 1 worker GPU (mínimo recomendado)
|
||||
docker compose up -d --scale translator-gpu=1
|
||||
|
||||
# Opción 2: Desplegar 2 workers GPU (recomendado: 16GB+ VRAM total)
|
||||
docker compose up -d --scale translator-gpu=2
|
||||
|
||||
# Opción 3: Desplegar 4 workers GPU (requiere: 32GB+ VRAM total)
|
||||
docker compose up -d --scale translator-gpu=4
|
||||
|
||||
# Opción 4: Combinar CPU y GPU
|
||||
# 2 workers CPU + 2 workers GPU
|
||||
docker compose up -d --scale translator=2 --scale translator-gpu=2
|
||||
```
|
||||
|
||||
**Panel de Administración:**
|
||||
Desde `/admin/settings`, puedes:
|
||||
- Activar modo GPU (CUDA) o CPU
|
||||
- Configurar número de workers (1-8)
|
||||
- Iniciar/detener workers automáticamente
|
||||
|
||||
**Configuración de GPU por Worker:**
|
||||
Cada worker GPU está configurado con:
|
||||
- `CT2_DEVICE=cuda` - Usa dispositivos CUDA
|
||||
- `CT2_COMPUTE_TYPE=float16` - Precisión mixta para velocidad
|
||||
- `TRANSLATOR_BATCH=128` - Tamaño de lote optimizado
|
||||
- `PYTORCH_CUDA_ALLOC_CONF=max_split_size_mb:512` - Gestión de memoria PyTorch
|
||||
- `NCCL_DEBUG=INFO` - Debug de comunicación multi-GPU
|
||||
|
||||
**Monitorización:**
|
||||
```bash
|
||||
# Ver estado de workers
|
||||
docker compose --profile gpu ps translator-gpu
|
||||
|
||||
# Ver uso de GPU
|
||||
watch -n 1 nvidia-smi
|
||||
|
||||
# Ver logs en tiempo real
|
||||
docker compose logs -f translator-gpu
|
||||
|
||||
# Métricas en Grafana
|
||||
# Dashboard: Translation Workers
|
||||
# Métricas: translator_*_transitions_per_minute
|
||||
```
|
||||
|
||||
**Performance Esperada:**
|
||||
| Workers GPU | VRAM Total | Tasa de Traducción |
|
||||
|-------------|------------|-------------------|
|
||||
| 1 worker | 8GB+ | ~500 traducciones/min |
|
||||
| 2 workers | 16GB+ | ~1000 traducciones/min |
|
||||
| 4 workers | 32GB+ | ~2000 traducciones/min |
|
||||
| 8 workers | 64GB+ | ~4000 traducciones/min |
|
||||
| Modo | CPU | RAM | Disco |
|
||||
|------|-----|-----|-------|
|
||||
| POC local | 2 cores | 4 GB | 10 GB |
|
||||
| Producción CPU | 4+ cores | 8 GB | 40 GB |
|
||||
| Producción recomendado | 8 cores | 16 GB | 80 GB |
|
||||
|
||||
---
|
||||
|
||||
## 🛡️ Administración y Mantenimiento
|
||||
|
||||
### Copias de Seguridad (Backups)
|
||||
|
||||
Desde el panel de Administración (`/admin/settings`), puedes realizar:
|
||||
|
||||
**Backup Completo (SQL):**
|
||||
- Descarga un archivo `.sql` con todo el contenido de la base de datos
|
||||
- Incluye: noticias, feeds, traducciones, usuarios, etiquetas, favoritos, videos, eventos, historial de búsqueda
|
||||
- Formato: PostgreSQL custom format con datos y estructura
|
||||
- Tamaño típico: 10-100MB dependiendo del volumen de datos
|
||||
|
||||
**Backup de Noticias (ZIP):**
|
||||
- Genera un archivo comprimido `.zip` con:
|
||||
- Tabla `noticias` (artículos completos)
|
||||
- Tabla `traducciones` (todas las traducciones)
|
||||
- Tablas `tags` y `tags_noticia` (metadatos y relaciones)
|
||||
- Ideal para: Migraciones de contenido, exportación de datos, backup ligero
|
||||
- Tamaño típico: 5-50MB (solo contenido de noticias)
|
||||
|
||||
### Variables de Entorno Clave (`.env`)
|
||||
| Variable | Descripción |
|
||||
|----------|-------------|
|
||||
| `WIKI_SLEEP` | Tiempo de espera entre peticiones a Wikipedia (evita bloqueos). |
|
||||
| `SCHEDULER_BATCH`| Cantidad de noticias a enviar a traducir por ciclo. |
|
||||
| `TARGET_LANGS` | Idiomas destino (ej: `es`). |
|
||||
| `OLLAMA_URL` | URL del servidor Ollama para categorización. |
|
||||
|
||||
---
|
||||
|
||||
## 🎯 Descripción de Workers
|
||||
|
||||
### rss-ingestor-go
|
||||
Crawler RSS de alto rendimiento escrito en Go. Gestiona hasta 100 workers paralelos para procesar múltiples feeds simultáneamente. Detecta nuevos artículos, extrae metadatos básicos y los inserta en la base de datos.
|
||||
|
||||
### scraper (Go)
|
||||
Scraper profundo que:
|
||||
- Descarga el contenido completo de URLs de artículos
|
||||
- Sanitiza HTML removiendo scripts y estilos
|
||||
- Extrae texto limpio del body
|
||||
- Limita el contenido a 20 noticias por ciclo para evitar sobrecarga
|
||||
|
||||
### discovery (Go)
|
||||
Agente de descubrimiento de feeds que:
|
||||
- Analiza URLs proporcionadas para detectar feeds RSS/Atom
|
||||
- Soporta hasta 5 feeds por URL
|
||||
- Ejecuta cada 900 segundos (15 minutos)
|
||||
|
||||
### translator (Python/CTranslate2)
|
||||
Worker de traducción neuronal:
|
||||
- Modelo: facebook/nllb-200-distilled-600M convertido a CTranslate2
|
||||
- Dispositivo: CPU (int8)
|
||||
- Batch: 32 traducciones por ciclo
|
||||
- Locking: `FOR UPDATE SKIP LOCKED` para evitar procesamiento duplicado
|
||||
- Soporta múltiples idiomas fuente
|
||||
|
||||
### translator-gpu (Python/CTranslate2)
|
||||
Versión acelerada por GPU del translator:
|
||||
- Dispositivo: CUDA
|
||||
- Compute type: float16
|
||||
- Batch: 128 traducciones por ciclo
|
||||
- Ideal para procesamiento de alto volumen
|
||||
|
||||
### remote-translator (Python)
|
||||
Worker remoto conectado vía WebSocket:
|
||||
- Autenticación con API key
|
||||
- Conexión a `ws://backend-go:8080/ws/worker`
|
||||
- Modelo: facebook/nllb-200-1.3B (más grande)
|
||||
- GPU requerida
|
||||
|
||||
### langdetect (Python)
|
||||
Detector de idioma:
|
||||
- Usa biblioteca `langdetect`
|
||||
- Proceso: 1000 noticias por ciclo
|
||||
- Establece lang_from para correcto routing a traducción
|
||||
|
||||
### ner (Python/spaCy)
|
||||
Extracción de entidades nombradas:
|
||||
- Modelo: es_core_news_lg
|
||||
- Tipos: PERSON (persona), ORG (organizacion), LOC/GPE (lugar)
|
||||
- También extrae topics (noun-chunks)
|
||||
- Configurable via entity_config.json (blacklist, synonyms)
|
||||
|
||||
### embeddings (Python)
|
||||
Generador de embeddings para búsqueda semántica:
|
||||
- Modelo: paraphrase-multilingual-MiniLM-L12-v2
|
||||
- Dispositivo: GPU (cuda)
|
||||
- Genera vectores de 384 dimensiones
|
||||
- Alimenta Qdrant
|
||||
|
||||
### qdrant-worker (Go)
|
||||
Worker de vectorización Qdrant:
|
||||
- Conecta a Qdrant (puerto 6333)
|
||||
- Genera embeddings via Ollama (mxbai-embed-large)
|
||||
- Sube puntos a collection "news_vectors"
|
||||
- Actualiza estado en BD (vectorized=true)
|
||||
|
||||
### related (Go)
|
||||
Detector de noticias relacionadas:
|
||||
- Usa similitud coseno entre embeddings
|
||||
- Almacena top-k relaciones en tabla related_noticias
|
||||
- Soporta configuración de threshold mínimo
|
||||
|
||||
### wiki-worker (Go)
|
||||
Enriquecedor de Wikipedia:
|
||||
- Descarga summaries de Wikipedia API
|
||||
- Descarga thumbnails de imágenes
|
||||
- Almacena en `./data/wiki_images`
|
||||
- Evita rate limiting con WIKI_SLEEP
|
||||
|
||||
### llm-categorizer (Python)
|
||||
Categorizador con LLM:
|
||||
- Conexión a Ollama
|
||||
- Usa modelo Mistral para clasificación
|
||||
- Procesa 10 noticias por batch
|
||||
|
||||
### cluster (Python)
|
||||
Agrupador de noticias por eventos:
|
||||
- Usa embeddings paraphrase-multilingual-MiniLM-L12-v2
|
||||
- Threshold: 0.35 de distancia
|
||||
- Agrupa noticias similares
|
||||
|
||||
### translation-scheduler (Python)
|
||||
Creador de tareas de traducción:
|
||||
- Query: noticias sin traducción al español
|
||||
- Crea entradas en tabla traducciones
|
||||
- Batch: 1000 por ciclo
|
||||
- Ciclo: 30 segundos
|
||||
|
||||
---
|
||||
|
||||
## 📖 Documentación de la API (Campos Wikipedia)
|
||||
|
||||
Las respuestas de noticias ahora incluyen el objeto `entities` enriquecido:
|
||||
|
||||
```json
|
||||
{
|
||||
"id": 67449,
|
||||
"titulo": "...",
|
||||
"entities": [
|
||||
{
|
||||
"valor": "Apple",
|
||||
"tipo": "organizacion",
|
||||
"wiki_summary": "Apple Inc. es una empresa estadounidense...",
|
||||
"wiki_url": "https://es.wikipedia.org/wiki/Apple",
|
||||
"image_path": "/api/wiki-images/wiki_5723.png"
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 🧪 Testing
|
||||
|
||||
### Backend (Go)
|
||||
```bash
|
||||
cd backend && go test ./internal/handlers -v -run TestLogin
|
||||
cd backend && go test ./internal/auth -v -run TestGenerateToken
|
||||
```
|
||||
|
||||
### Frontend
|
||||
```bash
|
||||
cd frontend && npm run test:ui
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 📁 Estructura de Archivos del Proyecto
|
||||
## Estructura del repositorio
|
||||
|
||||
```
|
||||
rss2/
|
||||
├── backend/ # API REST Go (Gin)
|
||||
├── frontend/ # Interfaz React + TypeScript
|
||||
├── workers/ # Workers Python
|
||||
├── rss-ingestor-go/ # Crawler RSS Go
|
||||
├── docker-compose.yml # Orquestación
|
||||
├── nginx.conf # Config Nginx
|
||||
├── monitoring/ # Prometheus + Grafana
|
||||
├── init-db/ # Migraciones SQL
|
||||
├── data/ # Datos persistentes
|
||||
├── models/ # Modelos ML
|
||||
├── hf_cache/ # Cache HuggingFace
|
||||
├── pre-deploy.sh # Script seguridad
|
||||
├── generate_secure_credentials.sh
|
||||
└── AGENTS.md # Guía desarrollo
|
||||
├── README.md
|
||||
├── requirements.txt Dependencias Python para workers ML
|
||||
├── Makefile Compilación local de binarios Go
|
||||
│
|
||||
├── backend/ Go — API REST + workers (scraper, discovery, wiki, topics, related, qdrant)
|
||||
├── rss-ingestor-go/ Go — Ingestor de feeds RSS
|
||||
├── frontend/ React + TypeScript + Tailwind
|
||||
├── workers/ Python — Workers ML (traducción, embeddings, NER, cluster, categorización)
|
||||
│
|
||||
├── init-db/ SQL — Schema completo y datos iniciales
|
||||
├── migrations/ SQL — Migraciones incrementales
|
||||
│
|
||||
├── config/
|
||||
│ └── entity_config.json Aliases y blacklist para normalización de entidades NER
|
||||
│
|
||||
├── data/
|
||||
│ └── feeds.csv Feeds RSS precargados para importar desde el admin
|
||||
│
|
||||
├── docs/
|
||||
│ ├── DEPLOY_DEBIAN.md Guía detallada de despliegue en Debian
|
||||
│ ├── SECURITY_GUIDE.md Guía de seguridad
|
||||
│ ├── SECURITY_AUDIT.md Resultado del audit de seguridad
|
||||
│ ├── QDRANT_SETUP.md Configuración de Qdrant
|
||||
│ └── ... Resto de documentación técnica
|
||||
│
|
||||
├── scripts/
|
||||
│ ├── generate_secure_credentials.sh
|
||||
│ ├── migrate_to_secure.sh
|
||||
│ └── verify_security.sh
|
||||
│
|
||||
├── deploy/debian/ Despliegue nativo en Debian (sin Docker)
|
||||
│ ├── prerequisites.sh Instala todas las dependencias del sistema
|
||||
│ ├── install.sh Instalación completa
|
||||
│ ├── build.sh Recompila y reinicia tras actualizar código
|
||||
│ ├── check.sh Diagnóstico del sistema
|
||||
│ ├── env.example Plantilla de variables de entorno
|
||||
│ ├── nginx.conf Configuración nginx
|
||||
│ └── systemd/ 16 ficheros de servicio systemd
|
||||
│
|
||||
└── poc/
|
||||
├── poc.sh POC local en 2 minutos (sin Docker, sin ML)
|
||||
└── seed.sql 10 noticias de muestra en español
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
**RSS2** - *Transformando noticias en inteligencia con IA localizada.*
|
||||
220
README_DEPLOY.md
220
README_DEPLOY.md
|
|
@ -1,220 +0,0 @@
|
|||
# 🚀 Despliegue RSS2 - 3 Comandos Simples
|
||||
|
||||
## ⚡ Flujo Rápido (3 pasos)
|
||||
|
||||
```bash
|
||||
# 1. Generar credenciales seguras
|
||||
./generate_secure_credentials.sh --force
|
||||
|
||||
# 2. Desplegar
|
||||
docker compose up -d
|
||||
|
||||
# 3. Verificar
|
||||
docker compose ps
|
||||
```
|
||||
|
||||
**¡Listo!** La aplicación estará disponible en `http://localhost:8888`
|
||||
|
||||
---
|
||||
|
||||
## 🔐 ¿Por qué 3 pasos?
|
||||
|
||||
**Problema original:**
|
||||
```bash
|
||||
docker compose up -d
|
||||
WARN: DB_PASS no definida → Contraseña vacía → ¡Inseguro!
|
||||
```
|
||||
|
||||
**Solución:**
|
||||
1. `./generate_secure_credentials.sh --force` → Crea `.env` con contraseñas de 32 caracteres
|
||||
2. `docker compose up -d` → Docker carga `.env` automáticamente
|
||||
3. `docker compose ps` → Verifica que todo esté bien
|
||||
|
||||
---
|
||||
|
||||
## 📋 Qué hace cada comando
|
||||
|
||||
### 1. `./generate_secure_credentials.sh --force`
|
||||
|
||||
- Crea contraseñas seguras de 32 caracteres aleatorios
|
||||
- Genera archivo `.env` con todas las variables necesarias
|
||||
- Guarda backup del `.env` anterior
|
||||
- **Tiempo: 5 segundos**
|
||||
|
||||
**Salida:**
|
||||
```
|
||||
POSTGRES_PASSWORD: S5xwsI9HKjdaBWLpCkqp0mA0DJYZerpD
|
||||
REDIS_PASSWORD: cVF79QuJulPO5auEhA2nNqv7mqAruzYh
|
||||
SECRET_KEY: 2eb7c221245bb8d896f83255188de614e59fae6b8dfbf16eee8a23c60dde4ffa
|
||||
GRAFANA_PASSWORD: R4J61V6OGLTCesrITwEvUPTm
|
||||
```
|
||||
|
||||
### 2. `docker compose up -d`
|
||||
|
||||
- Lee `.env` automáticamente (está configurado en `docker-compose.yml`)
|
||||
- Inicia todos los servicios en modo detached (background)
|
||||
- Espera a que estén listos
|
||||
- **Tiempo: 2-5 minutos** (primera vez que descarga imágenes)
|
||||
|
||||
### 3. `docker compose ps`
|
||||
|
||||
- Muestra estado de todos los servicios
|
||||
- Deberías ver `Up` para todos
|
||||
- **Tiempo: 1 segundo**
|
||||
|
||||
**Salida esperada:**
|
||||
```
|
||||
Name Status
|
||||
rss2_db Up (healthy)
|
||||
rss2_redis Up (healthy)
|
||||
rss2_backend_go Up (running)
|
||||
rss2_nginx Up (running)
|
||||
...
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 🎯 Comandos Útiles
|
||||
|
||||
### Verificar credenciales
|
||||
```bash
|
||||
grep -E "^(POSTGRES_PASSWORD|REDIS_PASSWORD|DB_PASS)=" .env
|
||||
```
|
||||
|
||||
### Ver logs en tiempo real
|
||||
```bash
|
||||
docker compose logs -f
|
||||
```
|
||||
|
||||
### Reiniciar servicios
|
||||
```bash
|
||||
# Todos
|
||||
docker compose restart
|
||||
|
||||
# Solo base de datos
|
||||
docker compose restart db
|
||||
```
|
||||
|
||||
### Verificar estado
|
||||
```bash
|
||||
docker compose ps
|
||||
```
|
||||
|
||||
### Limpiar todo (PERDERÁS DATOS)
|
||||
```bash
|
||||
docker compose down -v
|
||||
```
|
||||
|
||||
### Escalar workers
|
||||
```bash
|
||||
# Más traductores CPU
|
||||
docker compose up -d --scale translator=3
|
||||
|
||||
# Más traductores GPU
|
||||
docker compose up -d --scale translator-gpu=2
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 🔒 Seguridad
|
||||
|
||||
### Contraseñas por defecto = PELIGRO ❌
|
||||
|
||||
```bash
|
||||
# MAL - Contraseñas vacías
|
||||
docker compose up -d
|
||||
WARN: DB_PASS no set
|
||||
# Las contraseñas están vacías
|
||||
```
|
||||
|
||||
### Contraseñas generadas = SEGURO ✅
|
||||
|
||||
```bash
|
||||
# BIEN
|
||||
./generate_secure_credentials.sh --force
|
||||
docker compose up -d
|
||||
# Contraseñas de 32 caracteres aleatorios
|
||||
```
|
||||
|
||||
### Guardar tus credenciales
|
||||
|
||||
**Opción 1: Gestor de contraseñas (Recomendado)**
|
||||
```bash
|
||||
# Copia las contraseñas que muestra el script
|
||||
# Guárdalas en: LastPass, 1Password, KeePass, Bitwarden
|
||||
```
|
||||
|
||||
**Opción 2: Archivo seguro local**
|
||||
```bash
|
||||
# Crea un archivo fuera del proyecto
|
||||
echo "POSTGRES_PASSWORD=..." > ~/rss2-credentials.txt
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 📊 Estado de los Servicios
|
||||
|
||||
| Servicio | Puerto | Descripción |
|
||||
|----------|--------|-------------|
|
||||
| `nginx` | 8888 | Web app |
|
||||
| `backend-go` | 8080 | API |
|
||||
| `db` | 5432 | PostgreSQL |
|
||||
| `redis` | 6379 | Cache |
|
||||
| `qdrant` | 6333 | Vector DB |
|
||||
|
||||
**DASHBOARDS:**
|
||||
- Grafana: http://127.0.0.1:3001
|
||||
- Prometheus: http://127.0.0.1:9090
|
||||
|
||||
---
|
||||
|
||||
## 🐛 Solución de Problemas
|
||||
|
||||
### WARN sobre contraseñas vacías
|
||||
```bash
|
||||
# Solución: Generar credenciales
|
||||
./generate_secure_credentials.sh --force
|
||||
```
|
||||
|
||||
### Port 8888 en uso
|
||||
```bash
|
||||
# Cambiar puerto en nginx
|
||||
```
|
||||
|
||||
### Base de datos no inicia
|
||||
```bash
|
||||
# Ver logs
|
||||
docker compose logs db
|
||||
|
||||
# Reiniciar
|
||||
docker compose restart db
|
||||
```
|
||||
|
||||
### Redis autenticación fallida
|
||||
```bash
|
||||
# Ver REDIS_PASSWORD
|
||||
grep REDIS_PASSWORD .env
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## ✅ Checklist Final
|
||||
|
||||
Antes de considerar tu despliegue listo:
|
||||
|
||||
- [ ] `.env` tiene contraseñas generadas (no por defecto)
|
||||
- [ ] `docker compose ps` muestra todos "Up"
|
||||
- [ ] `http://localhost:8888` abre la app
|
||||
- [ ] `curl http://localhost:8888/api/health` responde
|
||||
|
||||
---
|
||||
|
||||
## 📚 Documentación
|
||||
|
||||
- [README.md](./README.md) - Documentación completa
|
||||
- [DEPLOY.md](./DEPLOY.md) - Guía detallada
|
||||
- [SECURITY_GUIDE.md](./SECURITY_GUIDE.md) - Seguridad
|
||||
|
||||
---
|
||||
|
||||
**RSS2** - *Transformando noticias en inteligencia con IA* 🚀
|
||||
128
alias.sh
128
alias.sh
|
|
@ -1,128 +0,0 @@
|
|||
#!/bin/bash
|
||||
# Aliases y funciones para despliegue automático de RSS2
|
||||
# Copia este archivo y ejecútalo: source alias.sh
|
||||
|
||||
# ========================================
|
||||
# FUNCIONES DE VALIDACIÓN AUTOMÁTICA
|
||||
# ========================================
|
||||
|
||||
# Validar credenciales críticas
|
||||
validate_creds() {
|
||||
# Verificar si .env existe
|
||||
if [ ! -f .env ]; then
|
||||
echo "🔑 Generando credenciales seguras..."
|
||||
./generate_secure_credentials.sh --force
|
||||
exit $?
|
||||
fi
|
||||
|
||||
# Verificar contraseñas vacías
|
||||
for var in POSTGRES_PASSWORD REDIS_PASSWORD DB_PASS; do
|
||||
val=$(grep "^${var}=" .env 2>/dev/null | cut -d'=' -f2 | tr -d ' ')
|
||||
if [ -z "$val" ] || [ "$val" = "change_*" ] || [ "$val" = "" ]; then
|
||||
echo "🔑 $var está vacía, generando..."
|
||||
./generate_secure_credentials.sh --force
|
||||
exit $?
|
||||
fi
|
||||
done
|
||||
|
||||
echo "✅ Credenciales OK"
|
||||
}
|
||||
|
||||
# ========================================
|
||||
# ALIASES PRINCIPALES
|
||||
# ========================================
|
||||
|
||||
# Alias principal: docker compose con validación automática
|
||||
alias dc="docker compose"
|
||||
alias dcu="docker_compose_up" # docker compose up
|
||||
alias dcd="docker_compose_down" # docker compose down
|
||||
|
||||
# Alias con validación automática
|
||||
docker_compose_up() {
|
||||
echo "🔐 Validando credenciales..."
|
||||
validate_creds
|
||||
|
||||
echo "🚀 docker compose up -d"
|
||||
dc up -d
|
||||
}
|
||||
|
||||
docker_compose_down() {
|
||||
echo "📦 docker compose down -v"
|
||||
dc down -v
|
||||
}
|
||||
|
||||
# Alias para despliegue completo con validación
|
||||
deploy() {
|
||||
echo "================================"
|
||||
echo "🚀 DESPLIEGUE AUTOMÁTICO RSS2"
|
||||
echo "================================"
|
||||
echo ""
|
||||
|
||||
echo "1. 🔐 Validando credenciales..."
|
||||
validate_creds
|
||||
|
||||
echo "2. 📦 Deteniendo y eliminando..."
|
||||
dc down -v 2>/dev/null || true
|
||||
|
||||
echo "3. 🚀 Iniciando servicios..."
|
||||
dc up -d --build
|
||||
|
||||
echo "4. ✅ Verificando estado..."
|
||||
dc ps
|
||||
|
||||
echo ""
|
||||
echo "================================"
|
||||
echo "✅ ¡Despliegue completado!"
|
||||
echo "🌐 App: http://localhost:8888"
|
||||
echo "================================"
|
||||
}
|
||||
|
||||
# Alias para despliegue rápido sin rebuild
|
||||
deploy-quick() {
|
||||
echo "🚀 Despliegue rápido (sin rebuild)..."
|
||||
dc up -d
|
||||
|
||||
echo "📊 Estado:"
|
||||
dc ps
|
||||
}
|
||||
|
||||
# Alias para ver logs
|
||||
tail-logs() {
|
||||
echo "📊 Logs en tiempo real..."
|
||||
dc logs -f
|
||||
}
|
||||
|
||||
# ========================================
|
||||
# FUNCIONES DE UTILIDAD
|
||||
# ========================================
|
||||
|
||||
# Generar credenciales
|
||||
generate-secrets() {
|
||||
echo "🔑 Generando credenciales seguras..."
|
||||
./generate_secure_credentials.sh --force
|
||||
}
|
||||
|
||||
# Backup
|
||||
backup() {
|
||||
echo "📦 Creando backup..."
|
||||
dc exec db pg_dump -U rss rss > backup_$(date +%Y%m%d).sql
|
||||
echo "✅ Backup guardado en ./backup_$(date +%Y%m%d).sql"
|
||||
}
|
||||
|
||||
# ========================================
|
||||
# MENSAJE DE BIENVENIDA
|
||||
# ========================================
|
||||
if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then
|
||||
echo "🎉 RSS2 Deployment Aliases cargadas"
|
||||
echo ""
|
||||
echo "Comandos disponibles:"
|
||||
echo " dc docker compose"
|
||||
echo " dcu docker compose up (con validación)"
|
||||
echo " dcd docker compose down"
|
||||
echo " deploy Despliegue completo con validación"
|
||||
echo " deploy-quick Despliegue rápido"
|
||||
echo " tail-logs Ver logs en tiempo real"
|
||||
echo " generate-secrets Generar credenciales"
|
||||
echo " backup Crear backup"
|
||||
echo ""
|
||||
fi
|
||||
|
|
@ -1,25 +0,0 @@
|
|||
FROM golang:1.23 AS builder
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
RUN apt-get update && apt-get install -y gcc musl-dev git
|
||||
|
||||
COPY go.mod ./
|
||||
RUN go mod download
|
||||
|
||||
COPY . .
|
||||
RUN go mod tidy
|
||||
|
||||
RUN CGO_ENABLED=0 GOOS=linux go build -buildvcs=false -o /server ./cmd/server
|
||||
|
||||
FROM alpine:3.23
|
||||
|
||||
RUN apk add --no-cache ca-certificates tzdata postgresql-client
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY --from=builder /server .
|
||||
|
||||
EXPOSE 8080
|
||||
|
||||
CMD ["./server"]
|
||||
|
|
@ -14,11 +14,11 @@ import (
|
|||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
"github.com/mmcdole/gofeed"
|
||||
"github.com/rss2/backend/internal/logger"
|
||||
"github.com/rss2/backend/internal/workers"
|
||||
)
|
||||
|
||||
var (
|
||||
logger *log.Logger
|
||||
pool *workers.Config
|
||||
dbPool *pgxpool.Pool
|
||||
sleepSec = 900 // 15 minutes
|
||||
|
|
@ -35,8 +35,7 @@ type URLSource struct {
|
|||
}
|
||||
|
||||
func init() {
|
||||
logLevel := os.Getenv("LOG_LEVEL")
|
||||
logger.Init("discovery-worker", logLevel)
|
||||
logger = log.New(os.Stdout, "[DISCOVERY] ", log.LstdFlags)
|
||||
}
|
||||
|
||||
func loadConfig() {
|
||||
|
|
@ -417,70 +416,48 @@ func processURLSource(ctx context.Context, source URLSource) {
|
|||
|
||||
func main() {
|
||||
loadConfig()
|
||||
logger.Info().Msg("Starting RSS Discovery Worker")
|
||||
logger.Println("Starting RSS Discovery Worker")
|
||||
|
||||
cfg := workers.LoadDBConfig()
|
||||
if err := workers.Connect(cfg); err != nil {
|
||||
logger.Fatal().Err(err).Msg("Failed to connect to database")
|
||||
logger.Fatalf("Failed to connect to database: %v", err)
|
||||
}
|
||||
dbPool = workers.GetPool()
|
||||
defer workers.Close()
|
||||
|
||||
logger.Info().Msg("Connected to PostgreSQL")
|
||||
logger.Println("Connected to PostgreSQL")
|
||||
|
||||
// Start health check HTTP server
|
||||
go func() {
|
||||
http.HandleFunc("/health", func(w http.ResponseWriter, r *http.Request) {
|
||||
if err := workers.HealthCheck(r.Context()); err != nil {
|
||||
http.Error(w, "unhealthy", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusOK)
|
||||
w.Write([]byte("ok"))
|
||||
})
|
||||
logger.Info().Msg("Health check server listening on :8080")
|
||||
if err := http.ListenAndServe(":8080", nil); err != nil {
|
||||
logger.Error().Err(err).Msg("Health check server error")
|
||||
}
|
||||
}()
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
ctx := context.Background()
|
||||
|
||||
sigChan := make(chan os.Signal, 1)
|
||||
signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM)
|
||||
|
||||
go func() {
|
||||
<-sigChan
|
||||
logger.Info().Msg("Shutting down gracefully...")
|
||||
cancel()
|
||||
time.Sleep(2 * time.Second)
|
||||
workers.Close()
|
||||
logger.Println("Shutting down...")
|
||||
os.Exit(0)
|
||||
}()
|
||||
|
||||
logger.Info().Msgf("Config: interval=%ds, batch=%d", sleepSec, batchSize)
|
||||
logger.Printf("Config: interval=%ds, batch=%d", sleepSec, batchSize)
|
||||
|
||||
ticker := time.NewTicker(time.Duration(sleepSec) * time.Second)
|
||||
defer ticker.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
logger.Info().Msg("Context cancelled, stopping worker")
|
||||
return
|
||||
case <-ticker.C:
|
||||
sources, err := getPendingURLs(ctx)
|
||||
if err != nil {
|
||||
logger.Error().Err(err).Msg("Error fetching URLs")
|
||||
logger.Printf("Error fetching URLs: %v", err)
|
||||
continue
|
||||
}
|
||||
|
||||
if len(sources) == 0 {
|
||||
logger.Info().Msg("No pending URLs to process")
|
||||
logger.Println("No pending URLs to process")
|
||||
continue
|
||||
}
|
||||
|
||||
logger.Info().Msgf("Processing %d sources", len(sources))
|
||||
logger.Printf("Processing %d sources", len(sources))
|
||||
|
||||
for _, source := range sources {
|
||||
processURLSource(ctx, source)
|
||||
|
|
|
|||
|
|
@ -6,52 +6,53 @@ import (
|
|||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/signal"
|
||||
"strconv"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
"github.com/rss2/backend/internal/logger"
|
||||
"github.com/rss2/backend/internal/workers"
|
||||
)
|
||||
|
||||
var (
|
||||
logger *log.Logger
|
||||
dbPool *pgxpool.Pool
|
||||
qdrantURL string
|
||||
ollamaURL string
|
||||
collection = "news_vectors"
|
||||
sleepSec = 30
|
||||
batchSize = 100
|
||||
)
|
||||
|
||||
func init() {
|
||||
logLevel := os.Getenv("LOG_LEVEL")
|
||||
logger.Init("qdrant-worker", logLevel)
|
||||
logger = log.New(os.Stdout, "[QDRANT] ", log.LstdFlags)
|
||||
}
|
||||
|
||||
func loadConfig() {
|
||||
sleepSec = getEnvInt("QDRANT_SLEEP", 30)
|
||||
batchSize = getEnvInt("QDRANT_BATCH",100)
|
||||
qdrantHost := getEnv("QDRANT_HOST","localhost")
|
||||
qdrantPort := getEnvInt("QDRANT_PORT",6333)
|
||||
qdrantURL = fmt.Sprintf("http://%s:%d",qdrantHost,qdrantPort)
|
||||
collection = getEnv("QDRANT_COLLECTION","news_vectors")
|
||||
batchSize = getEnvInt("QDRANT_BATCH", 100)
|
||||
qdrantHost := getEnv("QDRANT_HOST", "localhost")
|
||||
qdrantPort := getEnvInt("QDRANT_PORT", 6333)
|
||||
qdrantURL = fmt.Sprintf("http://%s:%d", qdrantHost, qdrantPort)
|
||||
ollamaURL = getEnv("OLLAMA_URL", "http://ollama:11434")
|
||||
collection = getEnv("QDRANT_COLLECTION", "news_vectors")
|
||||
}
|
||||
|
||||
func getEnv(key,defaultValue string) string {
|
||||
func getEnv(key, defaultValue string) string {
|
||||
if value := os.Getenv(key); value != "" {
|
||||
return value
|
||||
}
|
||||
return defaultValue
|
||||
}
|
||||
|
||||
func getEnvInt(key string,defaultValue int) int {
|
||||
func getEnvInt(key string, defaultValue int) int {
|
||||
if value := os.Getenv(key); value != "" {
|
||||
if intVal,err := strconv.Atoi(value); err == nil {
|
||||
if intVal, err := strconv.Atoi(value); err == nil {
|
||||
return intVal
|
||||
}
|
||||
}
|
||||
|
|
@ -60,7 +61,7 @@ func getEnvInt(key string,defaultValue int) int {
|
|||
|
||||
type Translation struct {
|
||||
ID int64
|
||||
NoticiaID string
|
||||
NoticiaID int64
|
||||
Lang string
|
||||
Titulo string
|
||||
Resumen string
|
||||
|
|
@ -69,7 +70,6 @@ type Translation struct {
|
|||
FuenteNombre string
|
||||
CategoriaID *int64
|
||||
PaisID *int64
|
||||
Embedding []float64
|
||||
}
|
||||
|
||||
func getPendingTranslations(ctx context.Context) ([]Translation, error) {
|
||||
|
|
@ -84,11 +84,9 @@ func getPendingTranslations(ctx context.Context) ([]Translation, error) {
|
|||
n.fecha,
|
||||
n.fuente_nombre,
|
||||
n.categoria_id,
|
||||
n.pais_id,
|
||||
te.embedding::text
|
||||
n.pais_id
|
||||
FROM traducciones t
|
||||
INNER JOIN noticias n ON t.noticia_id = n.id
|
||||
INNER JOIN traduccion_embeddings te ON te.traduccion_id = t.id
|
||||
WHERE t.vectorized = FALSE
|
||||
AND t.status = 'done'
|
||||
ORDER BY t.created_at ASC
|
||||
|
|
@ -102,41 +100,54 @@ func getPendingTranslations(ctx context.Context) ([]Translation, error) {
|
|||
var translations []Translation
|
||||
for rows.Next() {
|
||||
var t Translation
|
||||
var embText string
|
||||
if err := rows.Scan(
|
||||
&t.ID, &t.NoticiaID, &t.Lang, &t.Titulo, &t.Resumen,
|
||||
&t.URL, &t.Fecha, &t.FuenteNombre, &t.CategoriaID, &t.PaisID,
|
||||
&embText,
|
||||
); err != nil {
|
||||
logger.Printf("Error scanning translation row: %v", err)
|
||||
continue
|
||||
}
|
||||
emb, err := parseEmbedding(embText)
|
||||
if err != nil {
|
||||
logger.Printf("Error parsing embedding for translation %d: %v", t.ID, err)
|
||||
continue
|
||||
}
|
||||
t.Embedding = emb
|
||||
translations = append(translations, t)
|
||||
}
|
||||
return translations, nil
|
||||
}
|
||||
|
||||
func parseEmbedding(s string) ([]float64, error) {
|
||||
s = strings.Trim(strings.TrimSpace(s), "{}")
|
||||
if s == "" {
|
||||
return nil, fmt.Errorf("empty embedding")
|
||||
type EmbeddingRequest struct {
|
||||
Model string `json:"model"`
|
||||
Input string `json:"input"`
|
||||
}
|
||||
|
||||
type EmbeddingResponse struct {
|
||||
Embedding []float64 `json:"embedding"`
|
||||
}
|
||||
|
||||
func generateEmbedding(text string) ([]float64, error) {
|
||||
reqBody := EmbeddingRequest{
|
||||
Model: "mxbai-embed-large",
|
||||
Input: text,
|
||||
}
|
||||
parts := strings.Split(s, ",")
|
||||
emb := make([]float64, len(parts))
|
||||
for i, p := range parts {
|
||||
v, err := strconv.ParseFloat(strings.TrimSpace(p), 64)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
emb[i] = v
|
||||
|
||||
body, err := json.Marshal(reqBody)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return emb, nil
|
||||
|
||||
client := &http.Client{Timeout: 60 * time.Second}
|
||||
resp, err := client.Post(ollamaURL+"/api/embeddings", "application/json", bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, fmt.Errorf("Ollama returned status %d", resp.StatusCode)
|
||||
}
|
||||
|
||||
var result EmbeddingResponse
|
||||
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return result.Embedding, nil
|
||||
}
|
||||
|
||||
type QdrantPoint struct {
|
||||
|
|
@ -149,7 +160,7 @@ type QdrantUpsertRequest struct {
|
|||
Points []QdrantPoint `json:"points"`
|
||||
}
|
||||
|
||||
func ensureCollection(ctx context.Context) error {
|
||||
func ensureCollection() error {
|
||||
req, err := http.NewRequest("GET", qdrantURL+"/collections/"+collection, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
|
|
@ -166,15 +177,16 @@ func ensureCollection(ctx context.Context) error {
|
|||
return nil
|
||||
}
|
||||
|
||||
// Get embedding dimension from stored embeddings
|
||||
var dimension int
|
||||
err = dbPool.QueryRow(ctx, `SELECT dim FROM traduccion_embeddings LIMIT 1`).Scan(&dimension)
|
||||
// Get embedding dimension
|
||||
emb, err := generateEmbedding("test")
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get embedding dimension: %w", err)
|
||||
}
|
||||
dimension := len(emb)
|
||||
|
||||
// Create collection
|
||||
createReq := map[string]interface{}{
|
||||
"name": collection,
|
||||
"vectors": map[string]interface{}{
|
||||
"size": dimension,
|
||||
"distance": "Cosine",
|
||||
|
|
@ -182,35 +194,26 @@ func ensureCollection(ctx context.Context) error {
|
|||
}
|
||||
|
||||
body, _ := json.Marshal(createReq)
|
||||
createURL := qdrantURL + "/collections/" + collection
|
||||
req2, err := http.NewRequest(http.MethodPut, createURL, bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req2.Header.Set("Content-Type", "application/json")
|
||||
resp2, err := http.DefaultClient.Do(req2)
|
||||
resp2, err := http.Post(qdrantURL+"/collections", "application/json", bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp2.Body.Close()
|
||||
|
||||
if resp2.StatusCode != http.StatusOK {
|
||||
respBody, _ := io.ReadAll(resp2.Body)
|
||||
return fmt.Errorf("Qdrant create collection returned %d: %s", resp2.StatusCode, string(respBody))
|
||||
}
|
||||
|
||||
logger.Printf("Created collection %s with dimension %d", collection, dimension)
|
||||
return nil
|
||||
}
|
||||
|
||||
func uploadToQdrant(translations []Translation, pointIDs []string) error {
|
||||
func uploadToQdrant(translations []Translation, embeddings [][]float64) error {
|
||||
points := make([]QdrantPoint, 0, len(translations))
|
||||
|
||||
for i, t := range translations {
|
||||
if t.Embedding == nil || len(t.Embedding) == 0 {
|
||||
if embeddings[i] == nil {
|
||||
continue
|
||||
}
|
||||
|
||||
pointID := uuid.New().String()
|
||||
|
||||
payload := map[string]interface{}{
|
||||
"news_id": t.NoticiaID,
|
||||
"traduccion_id": t.ID,
|
||||
|
|
@ -232,8 +235,8 @@ func uploadToQdrant(translations []Translation, pointIDs []string) error {
|
|||
}
|
||||
|
||||
points = append(points, QdrantPoint{
|
||||
ID: pointIDs[i],
|
||||
Vector: t.Embedding,
|
||||
ID: pointID,
|
||||
Vector: embeddings[i],
|
||||
Payload: payload,
|
||||
})
|
||||
}
|
||||
|
|
@ -249,12 +252,7 @@ func uploadToQdrant(translations []Translation, pointIDs []string) error {
|
|||
}
|
||||
|
||||
url := fmt.Sprintf("%s/collections/%s/points", qdrantURL, collection)
|
||||
req, err := http.NewRequest(http.MethodPut, url, bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
resp, err := http.Post(url, "application/json", bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
@ -310,33 +308,17 @@ func main() {
|
|||
|
||||
cfg := workers.LoadDBConfig()
|
||||
if err := workers.Connect(cfg); err != nil {
|
||||
logger.Fatal().Err(err).Msg("Failed to connect to database")
|
||||
logger.Fatalf("Failed to connect to database: %v", err)
|
||||
}
|
||||
dbPool = workers.GetPool()
|
||||
defer workers.Close()
|
||||
|
||||
logger.Info().Msg("Connected to PostgreSQL")
|
||||
logger.Println("Connected to PostgreSQL")
|
||||
|
||||
// Start health check HTTP server
|
||||
go func() {
|
||||
http.HandleFunc("/health", func(w http.ResponseWriter, r *http.Request) {
|
||||
if err := workers.HealthCheck(r.Context()); err != nil {
|
||||
http.Error(w, "unhealthy", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusOK)
|
||||
w.Write([]byte("ok"))
|
||||
})
|
||||
logger.Info().Msg("Health check server listening on :8080")
|
||||
if err := http.ListenAndServe(":8080", nil); err != nil {
|
||||
logger.Error().Err(err).Msg("Health check server error")
|
||||
}
|
||||
}()
|
||||
ctx := context.Background()
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
|
||||
if err := ensureCollection(ctx); err != nil {
|
||||
logger.Warn().Err(err).Msg("Could not ensure collection")
|
||||
if err := ensureCollection(); err != nil {
|
||||
logger.Printf("Warning: Could not ensure collection: %v", err)
|
||||
}
|
||||
|
||||
sigChan := make(chan os.Signal, 1)
|
||||
|
|
@ -344,60 +326,65 @@ func main() {
|
|||
|
||||
go func() {
|
||||
<-sigChan
|
||||
logger.Info().Msg("Shutting down gracefully...")
|
||||
cancel()
|
||||
time.Sleep(2 * time.Second)
|
||||
workers.Close()
|
||||
logger.Println("Shutting down...")
|
||||
os.Exit(0)
|
||||
}()
|
||||
|
||||
logger.Info().Msgf("Config: qdrant=%s, collection=%s, sleep=%ds, batch=%d",
|
||||
qdrantURL, collection, sleepSec, batchSize)
|
||||
logger.Printf("Config: qdrant=%s, ollama=%s, collection=%s, sleep=%ds, batch=%d",
|
||||
qdrantURL, ollamaURL, collection, sleepSec, batchSize)
|
||||
|
||||
totalProcessed := 0
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
logger.Info().Msg("Context cancelled, stopping worker")
|
||||
return
|
||||
case <-time.After(time.Duration(sleepSec) * time.Second):
|
||||
translations, err := getPendingTranslations(ctx)
|
||||
if err != nil {
|
||||
logger.Error().Err(err).Msg("Error fetching pending translations")
|
||||
logger.Printf("Error fetching pending translations: %v", err)
|
||||
continue
|
||||
}
|
||||
|
||||
if len(translations) == 0 {
|
||||
logger.Info().Msg("No pending translations to process")
|
||||
logger.Println("No pending translations to process")
|
||||
continue
|
||||
}
|
||||
|
||||
logger.Info().Msgf("Processing %d translations...", len(translations))
|
||||
logger.Printf("Processing %d translations...", len(translations))
|
||||
|
||||
// Generate point IDs once (used both in Qdrant and DB)
|
||||
// Generate embeddings
|
||||
embeddings := make([][]float64, len(translations))
|
||||
for i, t := range translations {
|
||||
text := fmt.Sprintf("%s %s", t.Titulo, t.Resumen)
|
||||
emb, err := generateEmbedding(text)
|
||||
if err != nil {
|
||||
logger.Printf("Error generating embedding for %d: %v", t.ID, err)
|
||||
continue
|
||||
}
|
||||
embeddings[i] = emb
|
||||
}
|
||||
|
||||
// Upload to Qdrant
|
||||
if err := uploadToQdrant(translations, embeddings); err != nil {
|
||||
logger.Printf("Error uploading to Qdrant: %v", err)
|
||||
continue
|
||||
}
|
||||
|
||||
// Update DB status
|
||||
pointIDs := make([]string, len(translations))
|
||||
for i := range translations {
|
||||
pointIDs[i] = uuid.New().String()
|
||||
}
|
||||
|
||||
// Upload to Qdrant
|
||||
if err := uploadToQdrant(translations, pointIDs); err != nil {
|
||||
logger.Error().Err(err).Msg("Error uploading to Qdrant")
|
||||
continue
|
||||
}
|
||||
|
||||
// Update DB status
|
||||
if err := updateTranslationStatus(ctx, translations, pointIDs); err != nil {
|
||||
logger.Error().Err(err).Msg("Error updating status")
|
||||
logger.Printf("Error updating status: %v", err)
|
||||
}
|
||||
|
||||
totalProcessed += len(translations)
|
||||
logger.Info().Msgf("Processed %d translations (total: %d)", len(translations), totalProcessed)
|
||||
logger.Printf("Processed %d translations (total: %d)", len(translations), totalProcessed)
|
||||
|
||||
total, vectorized, pending, err := getStats(ctx)
|
||||
if err == nil {
|
||||
logger.Info().Msgf("Stats: total=%d, vectorized=%d, pending=%d", total, vectorized, pending)
|
||||
logger.Printf("Stats: total=%d, vectorized=%d, pending=%d", total, vectorized, pending)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3,7 +3,6 @@ package main
|
|||
import (
|
||||
"context"
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/signal"
|
||||
"strconv"
|
||||
|
|
@ -11,11 +10,11 @@ import (
|
|||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
"github.com/rss2/backend/internal/logger"
|
||||
"github.com/rss2/backend/internal/workers"
|
||||
)
|
||||
|
||||
var (
|
||||
logger *log.Logger
|
||||
dbPool *pgxpool.Pool
|
||||
sleepSec = 10
|
||||
topK = 10
|
||||
|
|
@ -24,8 +23,7 @@ var (
|
|||
)
|
||||
|
||||
func init() {
|
||||
logLevel := os.Getenv("LOG_LEVEL")
|
||||
logger.Init("related-worker", logLevel)
|
||||
logger = log.New(os.Stdout, "[RELATED] ", log.LstdFlags)
|
||||
}
|
||||
|
||||
func loadConfig() {
|
||||
|
|
@ -337,36 +335,20 @@ func processBatch(ctx context.Context, model string) (int, error) {
|
|||
|
||||
func main() {
|
||||
loadConfig()
|
||||
logger.Info().Msg("Starting Related News Worker")
|
||||
logger.Println("Starting Related News Worker")
|
||||
|
||||
cfg := workers.LoadDBConfig()
|
||||
if err := workers.Connect(cfg); err != nil {
|
||||
logger.Fatal().Err(err).Msg("Failed to connect to database")
|
||||
logger.Fatalf("Failed to connect to database: %v", err)
|
||||
}
|
||||
dbPool = workers.GetPool()
|
||||
defer workers.Close()
|
||||
|
||||
// Start health check HTTP server
|
||||
go func() {
|
||||
http.HandleFunc("/health", func(w http.ResponseWriter, r *http.Request) {
|
||||
if err := workers.HealthCheck(r.Context()); err != nil {
|
||||
http.Error(w, "unhealthy", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusOK)
|
||||
w.Write([]byte("ok"))
|
||||
})
|
||||
logger.Info().Msg("Health check server listening on :8080")
|
||||
if err := http.ListenAndServe(":8080", nil); err != nil {
|
||||
logger.Error().Err(err).Msg("Health check server error")
|
||||
}
|
||||
}()
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
ctx := context.Background()
|
||||
|
||||
// Ensure schema
|
||||
if err := ensureSchema(ctx); err != nil {
|
||||
logger.Error().Err(err).Msg("Error ensuring schema")
|
||||
logger.Printf("Error ensuring schema: %v", err)
|
||||
}
|
||||
|
||||
model := os.Getenv("EMB_MODEL")
|
||||
|
|
@ -379,29 +361,23 @@ func main() {
|
|||
|
||||
go func() {
|
||||
<-sigChan
|
||||
logger.Info().Msg("Shutting down gracefully...")
|
||||
cancel()
|
||||
time.Sleep(2 * time.Second)
|
||||
workers.Close()
|
||||
logger.Println("Shutting down...")
|
||||
os.Exit(0)
|
||||
}()
|
||||
|
||||
logger.Info().Msgf("Config: sleep=%ds, topK=%d, batch=%d, model=%s", sleepSec, topK, batchSz, model)
|
||||
logger.Printf("Config: sleep=%ds, topK=%d, batch=%d, model=%s", sleepSec, topK, batchSz, model)
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
logger.Info().Msg("Context cancelled, stopping worker")
|
||||
return
|
||||
case <-time.After(time.Duration(sleepSec) * time.Second):
|
||||
count, err := processBatch(ctx, model)
|
||||
if err != nil {
|
||||
logger.Error().Err(err).Msg("Error processing batch")
|
||||
logger.Printf("Error processing batch: %v", err)
|
||||
continue
|
||||
}
|
||||
|
||||
if count > 0 {
|
||||
logger.Info().Msgf("Generated related news for %d translations", count)
|
||||
logger.Printf("Generated related news for %d translations", count)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -4,29 +4,26 @@ import (
|
|||
"context"
|
||||
"crypto/md5"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/signal"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/PuerkitoBio/goquery"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
"github.com/rss2/backend/internal/logger"
|
||||
"github.com/rss2/backend/internal/workers"
|
||||
)
|
||||
|
||||
var (
|
||||
logger *log.Logger
|
||||
dbPool *workers.Config
|
||||
pool *pgxpool.Pool
|
||||
sleepInterval = 60
|
||||
batchSize = 10
|
||||
enrichLimit = 20
|
||||
scraperWorkers = 5
|
||||
maxBodyBytes = int64(512 << 10)
|
||||
)
|
||||
|
||||
type URLSource struct {
|
||||
|
|
@ -39,14 +36,6 @@ type URLSource struct {
|
|||
Active bool
|
||||
}
|
||||
|
||||
type Noticia struct {
|
||||
ID string
|
||||
Titulo string
|
||||
Resumen string
|
||||
URL string
|
||||
FuenteNombre string
|
||||
}
|
||||
|
||||
type Article struct {
|
||||
Title string
|
||||
Summary string
|
||||
|
|
@ -57,31 +46,13 @@ type Article struct {
|
|||
}
|
||||
|
||||
func init() {
|
||||
logLevel := os.Getenv("LOG_LEVEL")
|
||||
logger.Init("scraper-worker", logLevel)
|
||||
logger = log.New(os.Stdout, "[SCRAPER] ", log.LstdFlags)
|
||||
logger.SetOutput(os.Stdout)
|
||||
}
|
||||
|
||||
func loadConfig() {
|
||||
if v := os.Getenv("SCRAPER_INTERVAL"); v != "" {
|
||||
if i, err := strconv.Atoi(v); err == nil && i > 0 {
|
||||
sleepInterval = i
|
||||
}
|
||||
}
|
||||
if v := os.Getenv("SCRAPER_BATCH"); v != "" {
|
||||
if i, err := strconv.Atoi(v); err == nil && i > 0 {
|
||||
batchSize = i
|
||||
}
|
||||
}
|
||||
if v := os.Getenv("SCRAPER_ENRICH_LIMIT"); v != "" {
|
||||
if i, err := strconv.Atoi(v); err == nil && i > 0 {
|
||||
enrichLimit = i
|
||||
}
|
||||
}
|
||||
if v := os.Getenv("SCRAPER_WORKERS"); v != "" {
|
||||
if i, err := strconv.Atoi(v); err == nil && i > 0 {
|
||||
scraperWorkers = i
|
||||
}
|
||||
}
|
||||
sleepInterval = getEnvInt("SCRAPER_SLEEP", 60)
|
||||
batchSize = getEnvInt("SCRAPER_BATCH", 10)
|
||||
}
|
||||
|
||||
func getEnvInt(key string, defaultValue int) int {
|
||||
|
|
@ -95,9 +66,9 @@ func getEnvInt(key string, defaultValue int) int {
|
|||
|
||||
func getActiveURLs(ctx context.Context) ([]URLSource, error) {
|
||||
rows, err := pool.Query(ctx, `
|
||||
SELECT id, nombre, url, categoria_id, pais_id, idioma, active
|
||||
FROM fuentes_url
|
||||
WHERE active = true
|
||||
SELECT id, nombre, url, categoria_id, pais_id, idioma, activo
|
||||
FROM fuentes_url
|
||||
WHERE activo = true
|
||||
`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
|
@ -118,49 +89,16 @@ func getActiveURLs(ctx context.Context) ([]URLSource, error) {
|
|||
|
||||
func updateSourceStatus(ctx context.Context, sourceID int64, status, message string, httpCode int) error {
|
||||
_, err := pool.Exec(ctx, `
|
||||
UPDATE fuentes_url
|
||||
SET last_check = NOW(),
|
||||
last_status = $1,
|
||||
status_message = $2,
|
||||
last_http_code = $3
|
||||
WHERE id = $4
|
||||
UPDATE fuentes_url
|
||||
SET last_check = NOW(),
|
||||
last_status = $1,
|
||||
status_message = $2,
|
||||
last_http_code = $3
|
||||
WHERE id = $4
|
||||
`, status, message, httpCode, sourceID)
|
||||
return err
|
||||
}
|
||||
|
||||
func getNoticiasToEnrich(ctx context.Context, limit int) ([]Noticia, error) {
|
||||
rows, err := pool.Query(ctx, `
|
||||
SELECT id, titulo, COALESCE(resumen, ''), url, fuente_nombre
|
||||
FROM noticias
|
||||
WHERE (resumen IS NULL OR LENGTH(resumen) < 200)
|
||||
ORDER BY fecha DESC
|
||||
LIMIT $1
|
||||
`, limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var noticias []Noticia
|
||||
for rows.Next() {
|
||||
var n Noticia
|
||||
if err := rows.Scan(&n.ID, &n.Titulo, &n.Resumen, &n.URL, &n.FuenteNombre); err != nil {
|
||||
continue
|
||||
}
|
||||
noticias = append(noticias, n)
|
||||
}
|
||||
return noticias, nil
|
||||
}
|
||||
|
||||
func updateNoticiaResumen(ctx context.Context, noticiaID, newResumen string) error {
|
||||
_, err := pool.Exec(ctx, `
|
||||
UPDATE noticias
|
||||
SET resumen = $1
|
||||
WHERE id = $2
|
||||
`, newResumen, noticiaID)
|
||||
return err
|
||||
}
|
||||
|
||||
func extractArticle(source URLSource) (*Article, error) {
|
||||
client := &http.Client{
|
||||
Timeout: 30 * time.Second,
|
||||
|
|
@ -185,7 +123,7 @@ func extractArticle(source URLSource) (*Article, error) {
|
|||
return nil, fmt.Errorf("HTTP %d", resp.StatusCode)
|
||||
}
|
||||
|
||||
doc, err := goquery.NewDocumentFromReader(io.LimitReader(resp.Body, maxBodyBytes))
|
||||
doc, err := goquery.NewDocumentFromReader(resp.Body)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
|
@ -248,94 +186,6 @@ func extractArticle(source URLSource) (*Article, error) {
|
|||
return article, nil
|
||||
}
|
||||
|
||||
func extractContentFromURL(url string) (string, error) {
|
||||
client := &http.Client{
|
||||
Timeout: 30 * time.Second,
|
||||
}
|
||||
|
||||
req, err := http.NewRequest("GET", url, nil)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36")
|
||||
req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")
|
||||
req.Header.Set("Accept-Language", "en-US,en;q=0.5")
|
||||
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != 200 {
|
||||
return "", fmt.Errorf("HTTP %d", resp.StatusCode)
|
||||
}
|
||||
|
||||
doc, err := goquery.NewDocumentFromReader(io.LimitReader(resp.Body, maxBodyBytes))
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
contentSelectors := []string{
|
||||
"article",
|
||||
"[role='main']",
|
||||
"main",
|
||||
".article-content",
|
||||
".post-content",
|
||||
".entry-content",
|
||||
".content",
|
||||
"#content",
|
||||
".story-body",
|
||||
".article-body",
|
||||
}
|
||||
|
||||
var content string
|
||||
for _, sel := range contentSelectors {
|
||||
content = doc.Find(sel).First().Text()
|
||||
if len(content) > 100 {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if len(content) < 100 {
|
||||
ps := doc.Find("p")
|
||||
var paragraphs []string
|
||||
ps.Each(func(i int, s *goquery.Selection) {
|
||||
txt := strings.TrimSpace(s.Text())
|
||||
if len(txt) > 50 {
|
||||
paragraphs = append(paragraphs, txt)
|
||||
}
|
||||
})
|
||||
content = strings.Join(paragraphs, "\n\n")
|
||||
}
|
||||
|
||||
return strings.TrimSpace(content), nil
|
||||
}
|
||||
|
||||
func processEnrichment(ctx context.Context, noticia Noticia) bool {
|
||||
logger.Printf("Enriching: %s", noticia.Titulo)
|
||||
|
||||
content, err := extractContentFromURL(noticia.URL)
|
||||
if err != nil {
|
||||
logger.Printf("Error extracting content from %s: %v", noticia.URL, err)
|
||||
return false
|
||||
}
|
||||
|
||||
if len(content) < 50 {
|
||||
logger.Printf("Not enough content extracted from %s", noticia.URL)
|
||||
return false
|
||||
}
|
||||
|
||||
if err := updateNoticiaResumen(ctx, noticia.ID, content); err != nil {
|
||||
logger.Printf("Error updating resumen for %s: %v", noticia.ID, err)
|
||||
return false
|
||||
}
|
||||
|
||||
logger.Printf("Enriched: %s (content length: %d)", noticia.Titulo, len(content))
|
||||
return true
|
||||
}
|
||||
|
||||
func saveArticle(ctx context.Context, source URLSource, article *Article) (bool, error) {
|
||||
finalURL := article.URL
|
||||
if finalURL == "" {
|
||||
|
|
@ -425,71 +275,20 @@ func processSource(ctx context.Context, source URLSource) {
|
|||
}
|
||||
}
|
||||
|
||||
// enrichConcurrent enriches noticias in parallel with a bounded worker pool.
|
||||
func enrichConcurrent(ctx context.Context, noticias []Noticia) {
|
||||
limiter := make(chan struct{}, scraperWorkers)
|
||||
var wg sync.WaitGroup
|
||||
for _, noticia := range noticias {
|
||||
wg.Add(1)
|
||||
limiter <- struct{}{}
|
||||
go func(n Noticia) {
|
||||
defer wg.Done()
|
||||
defer func() { <-limiter }()
|
||||
if processEnrichment(ctx, n) {
|
||||
time.Sleep(1 * time.Second)
|
||||
}
|
||||
}(noticia)
|
||||
}
|
||||
wg.Wait()
|
||||
}
|
||||
|
||||
// scrapeSources processes fuentes_url in parallel with a bounded worker pool.
|
||||
func scrapeSources(ctx context.Context, sources []URLSource) {
|
||||
limiter := make(chan struct{}, scraperWorkers)
|
||||
var wg sync.WaitGroup
|
||||
for _, source := range sources {
|
||||
wg.Add(1)
|
||||
limiter <- struct{}{}
|
||||
go func(src URLSource) {
|
||||
defer wg.Done()
|
||||
defer func() { <-limiter }()
|
||||
time.Sleep(1 * time.Second) // polite pacing across workers
|
||||
processSource(ctx, src)
|
||||
}(source)
|
||||
}
|
||||
wg.Wait()
|
||||
}
|
||||
|
||||
func main() {
|
||||
loadConfig()
|
||||
logger.Info().Msg("Starting Scraper Worker")
|
||||
logger.Println("Starting Scraper Worker")
|
||||
|
||||
cfg := workers.LoadDBConfig()
|
||||
if err := workers.Connect(cfg); err != nil {
|
||||
logger.Fatal().Err(err).Msg("Failed to connect to database")
|
||||
logger.Fatalf("Failed to connect to database: %v", err)
|
||||
}
|
||||
pool = workers.GetPool()
|
||||
defer workers.Close()
|
||||
|
||||
logger.Info().Msg("Connected to PostgreSQL")
|
||||
logger.Println("Connected to PostgreSQL")
|
||||
|
||||
// Start health check HTTP server
|
||||
go func() {
|
||||
http.HandleFunc("/health", func(w http.ResponseWriter, r *http.Request) {
|
||||
if err := workers.HealthCheck(r.Context()); err != nil {
|
||||
http.Error(w, "unhealthy", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusOK)
|
||||
w.Write([]byte("ok"))
|
||||
})
|
||||
logger.Info().Msg("Health check server listening on :8080")
|
||||
if err := http.ListenAndServe(":8080", nil); err != nil {
|
||||
logger.Error().Err(err).Msg("Health check server error")
|
||||
}
|
||||
}()
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
ctx := context.Background()
|
||||
|
||||
// Handle shutdown
|
||||
sigChan := make(chan os.Signal, 1)
|
||||
|
|
@ -497,45 +296,35 @@ func main() {
|
|||
|
||||
go func() {
|
||||
<-sigChan
|
||||
logger.Info().Msg("Shutting down gracefully...")
|
||||
cancel()
|
||||
time.Sleep(2 * time.Second)
|
||||
workers.Close()
|
||||
logger.Println("Shutting down...")
|
||||
os.Exit(0)
|
||||
}()
|
||||
|
||||
logger.Info().Msgf("Config: sleep=%ds, batch=%d, enrich=%d", sleepInterval, batchSize, enrichLimit)
|
||||
logger.Printf("Config: sleep=%ds, batch=%d", sleepInterval, batchSize)
|
||||
|
||||
ticker := time.NewTicker(time.Duration(sleepInterval) * time.Second)
|
||||
defer ticker.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
logger.Info().Msg("Context cancelled, stopping worker")
|
||||
return
|
||||
case <-ticker.C:
|
||||
noticias, err := getNoticiasToEnrich(ctx, enrichLimit)
|
||||
if err != nil {
|
||||
logger.Error().Err(err).Msg("Error fetching noticias to enrich")
|
||||
} else if len(noticias) > 0 {
|
||||
logger.Info().Msgf("Enriching %d noticias (%d workers)", len(noticias), scraperWorkers)
|
||||
enrichConcurrent(ctx, noticias)
|
||||
}
|
||||
|
||||
sources, err := getActiveURLs(ctx)
|
||||
if err != nil {
|
||||
logger.Error().Err(err).Msg("Error fetching URLs")
|
||||
logger.Printf("Error fetching URLs: %v", err)
|
||||
continue
|
||||
}
|
||||
|
||||
if len(sources) == 0 {
|
||||
logger.Info().Msg("No active URLs to process")
|
||||
logger.Println("No active URLs to process")
|
||||
continue
|
||||
}
|
||||
|
||||
logger.Info().Msgf("Processing %d sources (%d workers)", len(sources), scraperWorkers)
|
||||
scrapeSources(ctx, sources)
|
||||
logger.Printf("Processing %d sources", len(sources))
|
||||
|
||||
for _, source := range sources {
|
||||
processSource(ctx, source)
|
||||
time.Sleep(2 * time.Second) // Rate limiting
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3,27 +3,22 @@ package main
|
|||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"os/signal"
|
||||
"syscall"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/rss2/backend/internal/auth"
|
||||
"github.com/rss2/backend/internal/cache"
|
||||
"github.com/rss2/backend/internal/config"
|
||||
"github.com/rss2/backend/internal/db"
|
||||
"github.com/rss2/backend/internal/handlers"
|
||||
"github.com/rss2/backend/internal/logger"
|
||||
"github.com/rss2/backend/internal/middleware"
|
||||
"github.com/rss2/backend/internal/services"
|
||||
swaggerFiles "github.com/swaggo/files"
|
||||
ginSwagger "github.com/swaggo/gin-swagger"
|
||||
_ "github.com/rss2/backend/docs"
|
||||
)
|
||||
|
||||
func initDB() {
|
||||
ctx := context.Background()
|
||||
log := logger.GetLogger()
|
||||
|
||||
// Crear tabla entity_aliases si no existe
|
||||
_, err := db.GetPool().Exec(ctx, `
|
||||
|
|
@ -37,9 +32,9 @@ func initDB() {
|
|||
)
|
||||
`)
|
||||
if err != nil {
|
||||
log.Warn().Err(err).Msg("Could not create entity_aliases table")
|
||||
log.Printf("Warning: Could not create entity_aliases table: %v", err)
|
||||
} else {
|
||||
log.Info().Msg("Table entity_aliases ready")
|
||||
log.Println("Table entity_aliases ready")
|
||||
}
|
||||
|
||||
// Añadir columna role a users si no existe
|
||||
|
|
@ -47,9 +42,9 @@ func initDB() {
|
|||
ALTER TABLE users ADD COLUMN IF NOT EXISTS role VARCHAR(20) DEFAULT 'user'
|
||||
`)
|
||||
if err != nil {
|
||||
log.Warn().Err(err).Msg("Could not add role column")
|
||||
log.Printf("Warning: Could not add role column: %v", err)
|
||||
} else {
|
||||
log.Info().Msg("Column role ready")
|
||||
log.Println("Column role ready")
|
||||
}
|
||||
|
||||
// Crear tabla de configuración si no existe
|
||||
|
|
@ -79,120 +74,31 @@ func initDB() {
|
|||
INSERT INTO config (key, value) VALUES ('translator_status', 'stopped')
|
||||
ON CONFLICT (key) DO NOTHING
|
||||
`)
|
||||
|
||||
// Crear tabla de alertas si no existe
|
||||
_, err = db.GetPool().Exec(ctx, `
|
||||
CREATE TABLE IF NOT EXISTS alertas (
|
||||
id SERIAL PRIMARY KEY,
|
||||
valor VARCHAR(255) NOT NULL,
|
||||
tipo VARCHAR(32) NOT NULL,
|
||||
periodo DATE NOT NULL,
|
||||
hits INT NOT NULL,
|
||||
baseline DOUBLE PRECISION NOT NULL,
|
||||
ratio DOUBLE PRECISION NOT NULL,
|
||||
status VARCHAR(16) NOT NULL DEFAULT 'nueva',
|
||||
created_at TIMESTAMP DEFAULT NOW(),
|
||||
UNIQUE(valor, tipo, periodo)
|
||||
)
|
||||
`)
|
||||
if err != nil {
|
||||
log.Printf("Warning: Could not create alertas table: %v", err)
|
||||
} else {
|
||||
log.Println("Table alertas ready")
|
||||
}
|
||||
|
||||
// Crear tabla de remote_workers si no existe
|
||||
_, err = db.GetPool().Exec(ctx, `
|
||||
CREATE TABLE IF NOT EXISTS remote_workers (
|
||||
id SERIAL PRIMARY KEY,
|
||||
name VARCHAR(255) NOT NULL,
|
||||
api_key VARCHAR(64) UNIQUE NOT NULL,
|
||||
capabilities VARCHAR(50) DEFAULT 'cpu',
|
||||
status VARCHAR(20) DEFAULT 'offline',
|
||||
last_seen TIMESTAMP,
|
||||
created_at TIMESTAMP DEFAULT NOW()
|
||||
)
|
||||
`)
|
||||
if err != nil {
|
||||
log.Printf("Warning: Could not create remote_workers table: %v", err)
|
||||
} else {
|
||||
log.Println("Table remote_workers ready")
|
||||
}
|
||||
|
||||
// Añadir columnas a traducciones para workers remotos
|
||||
_, err = db.GetPool().Exec(ctx, `
|
||||
ALTER TABLE traducciones ADD COLUMN IF NOT EXISTS worker_id INTEGER REFERENCES remote_workers(id)
|
||||
`)
|
||||
if err != nil {
|
||||
log.Printf("Warning: Could not add worker_id column: %v", err)
|
||||
}
|
||||
|
||||
_, err = db.GetPool().Exec(ctx, `
|
||||
ALTER TABLE traducciones ADD COLUMN IF NOT EXISTS assigned_at TIMESTAMP
|
||||
`)
|
||||
if err != nil {
|
||||
log.Printf("Warning: Could not add assigned_at column: %v", err)
|
||||
}
|
||||
|
||||
// Crear índices para optimizar consultas críticas
|
||||
indexes := []string{
|
||||
"CREATE INDEX IF NOT EXISTS idx_noticias_fecha ON noticias(fecha DESC)",
|
||||
"CREATE INDEX IF NOT EXISTS idx_noticias_fuente_nombre ON noticias(fuente_nombre)",
|
||||
"CREATE INDEX IF NOT EXISTS idx_noticias_categoria_id ON noticias(categoria_id)",
|
||||
"CREATE INDEX IF NOT EXISTS idx_noticias_pais_id ON noticias(pais_id)",
|
||||
"CREATE INDEX IF NOT EXISTS idx_noticias_lang ON noticias(lang)",
|
||||
"CREATE INDEX IF NOT EXISTS idx_traducciones_noticia_id_lang_to ON traducciones(noticia_id, lang_to)",
|
||||
"CREATE INDEX IF NOT EXISTS idx_traducciones_status ON traducciones(status)",
|
||||
"CREATE INDEX IF NOT EXISTS idx_tags_noticia_noticia_id_tag_id ON tags_noticia(noticia_id, tag_id)",
|
||||
"CREATE INDEX IF NOT EXISTS idx_tags_noticia_traduccion_id ON tags_noticia(traduccion_id)",
|
||||
"CREATE INDEX IF NOT EXISTS idx_tags_valor_tipo ON tags(valor, tipo)",
|
||||
"CREATE INDEX IF NOT EXISTS idx_entity_aliases_alias_tipo ON entity_aliases(alias, tipo)",
|
||||
"CREATE INDEX IF NOT EXISTS idx_alertas_valor_tipo_periodo ON alertas(valor, tipo, periodo)",
|
||||
"CREATE INDEX IF NOT EXISTS idx_alertas_status ON alertas(status)",
|
||||
"CREATE INDEX IF NOT EXISTS idx_user_search_tags_user_id ON user_search_tags(user_id)",
|
||||
"CREATE INDEX IF NOT EXISTS idx_feeds_activo ON feeds(activo)",
|
||||
"CREATE INDEX IF NOT EXISTS idx_feeds_categoria_id ON feeds(categoria_id)",
|
||||
"CREATE INDEX IF NOT EXISTS idx_feeds_pais_id ON feeds(pais_id)",
|
||||
}
|
||||
for _, idx := range indexes {
|
||||
_, err := db.GetPool().Exec(ctx, idx)
|
||||
if err != nil {
|
||||
log.Warn().Err(err).Msg("Could not create index")
|
||||
} else {
|
||||
log.Info().Msg("Index created/verified")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func main() {
|
||||
cfg := config.Load()
|
||||
|
||||
// Initialize structured logger
|
||||
logLevel := os.Getenv("LOG_LEVEL")
|
||||
logger.Init("rss2-api", logLevel)
|
||||
log := logger.GetLogger()
|
||||
|
||||
if err := db.Connect(cfg.DatabaseURL); err != nil {
|
||||
log.Fatal().Err(err).Msg("Failed to connect to database")
|
||||
log.Fatalf("Failed to connect to database: %v", err)
|
||||
}
|
||||
defer db.Close()
|
||||
log.Info().Msg("Connected to PostgreSQL")
|
||||
log.Println("Connected to PostgreSQL")
|
||||
|
||||
// Auto-setup DB tables
|
||||
initDB()
|
||||
|
||||
if err := cache.Connect(cfg.RedisURL); err != nil {
|
||||
log.Warn().Err(err).Msg("Failed to connect to Redis")
|
||||
log.Printf("Warning: Failed to connect to Redis: %v", err)
|
||||
} else {
|
||||
defer cache.Close()
|
||||
log.Info().Msg("Connected to Redis")
|
||||
log.Println("Connected to Redis")
|
||||
}
|
||||
|
||||
services.Init(cfg)
|
||||
|
||||
r := gin.Default()
|
||||
|
||||
r.Use(middleware.RequestIDMiddleware())
|
||||
r.Use(middleware.CORSMiddleware())
|
||||
r.Use(middleware.LoggerMiddleware())
|
||||
|
||||
|
|
@ -200,25 +106,18 @@ func main() {
|
|||
c.JSON(200, gin.H{"status": "ok"})
|
||||
})
|
||||
|
||||
// Swagger documentation
|
||||
r.GET("/swagger/*any", ginSwagger.WrapHandler(swaggerFiles.Handler))
|
||||
|
||||
api := r.Group("/api")
|
||||
{
|
||||
// Serve static images downloaded by wiki_worker
|
||||
api.StaticFS("/wiki-images", gin.Dir(cfg.WikiImagesPath, false))
|
||||
|
||||
// Stricter rate limiting for auth endpoints
|
||||
authGroup := api.Group("/auth")
|
||||
authGroup.Use(middleware.RateLimitMiddleware(10)) // 10 req/min for auth
|
||||
{
|
||||
authGroup.POST("/login", handlers.Login)
|
||||
authGroup.POST("/register", handlers.Register)
|
||||
authGroup.GET("/check-first-user", handlers.CheckFirstUser)
|
||||
wikiImagesDir := os.Getenv("WIKI_IMAGES_PATH")
|
||||
if wikiImagesDir == "" {
|
||||
wikiImagesDir = "/opt/rss2/data/wiki_images"
|
||||
}
|
||||
api.StaticFS("/wiki-images", gin.Dir(wikiImagesDir, false))
|
||||
|
||||
// General rate limiting for API
|
||||
api.Use(middleware.RateLimitMiddleware(cfg.RateLimitPerMinute))
|
||||
api.POST("/auth/login", handlers.Login)
|
||||
api.POST("/auth/register", handlers.Register)
|
||||
api.GET("/auth/check-first-user", handlers.CheckFirstUser)
|
||||
|
||||
news := api.Group("/news")
|
||||
{
|
||||
|
|
@ -241,16 +140,8 @@ func main() {
|
|||
}
|
||||
|
||||
api.GET("/search", handlers.SearchNews)
|
||||
api.GET("/search/suggestions", middleware.AuthRequired(), handlers.SearchSuggestions)
|
||||
api.POST("/searchlog", middleware.AuthRequired(), handlers.LogSearch)
|
||||
|
||||
api.GET("/entities", handlers.GetEntities)
|
||||
api.GET("/entities/news", handlers.GetEntityNews)
|
||||
api.GET("/entities/mentions", handlers.GetEntityMentions)
|
||||
|
||||
api.GET("/alerts", handlers.GetAlertas)
|
||||
api.POST("/alerts/:id/read", middleware.AuthRequired(), handlers.MarkAlertaRead)
|
||||
api.POST("/alerts/read-all", middleware.AuthRequired(), handlers.MarkAllAlertasRead)
|
||||
|
||||
api.GET("/stats", handlers.GetStats)
|
||||
|
||||
|
|
@ -266,28 +157,16 @@ func main() {
|
|||
admin.POST("/entities/retype", handlers.PatchEntityTipo)
|
||||
admin.GET("/backup", handlers.BackupDatabase)
|
||||
admin.GET("/backup/news", handlers.BackupNewsZipped)
|
||||
admin.POST("/restore", handlers.RestoreDatabase)
|
||||
admin.GET("/users", handlers.GetUsers)
|
||||
admin.POST("/users/:id/promote", handlers.PromoteUser)
|
||||
admin.POST("/users/:id/demote", handlers.DemoteUser)
|
||||
admin.POST("/reset-db", handlers.ResetDatabase)
|
||||
admin.POST("/alerts/scan", handlers.ScanAlertasAdmin)
|
||||
admin.GET("/workers/status", handlers.GetWorkerStatus)
|
||||
admin.GET("/workers/stats", handlers.GetTranslationStats)
|
||||
admin.POST("/workers/config", handlers.SetWorkerConfig)
|
||||
admin.POST("/workers/start", handlers.StartWorkers)
|
||||
admin.POST("/workers/stop", handlers.StopWorkers)
|
||||
|
||||
admin.GET("/workers/remote", handlers.ListRemoteWorkers)
|
||||
admin.POST("/workers/remote", handlers.CreateRemoteWorker)
|
||||
admin.GET("/workers/remote/:id", handlers.GetRemoteWorker)
|
||||
admin.DELETE("/workers/remote/:id", handlers.DeleteRemoteWorker)
|
||||
admin.POST("/workers/remote/:id/toggle", handlers.ToggleRemoteWorker)
|
||||
admin.POST("/workers/remote/:id/regenerate-key", handlers.RegenerateAPIKey)
|
||||
}
|
||||
|
||||
r.GET("/ws/worker", handlers.HandleWorkerWS)
|
||||
|
||||
auth := api.Group("/auth")
|
||||
auth.Use(middleware.AuthRequired())
|
||||
{
|
||||
|
|
@ -295,24 +174,21 @@ func main() {
|
|||
}
|
||||
}
|
||||
|
||||
auth.SetJWTSecret(cfg.SecretKey)
|
||||
middleware.SetJWTSecret(cfg.SecretKey)
|
||||
|
||||
port := cfg.ServerPort
|
||||
addr := fmt.Sprintf(":%s", port)
|
||||
|
||||
go func() {
|
||||
log.Info().Str("addr", addr).Msg("Server starting")
|
||||
log.Printf("Server starting on %s", addr)
|
||||
if err := r.Run(addr); err != nil {
|
||||
log.Fatal().Err(err).Msg("Failed to start server")
|
||||
log.Fatalf("Failed to start server: %v", err)
|
||||
}
|
||||
}()
|
||||
|
||||
handlers.StartJobAssigner()
|
||||
handlers.StartAlertScanner()
|
||||
|
||||
quit := make(chan os.Signal, 1)
|
||||
signal.Notify(quit, syscall.SIGINT, syscall.SIGTERM)
|
||||
<-quit
|
||||
|
||||
log.Info().Msg("Shutting down server...")
|
||||
log.Println("Shutting down server...")
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2,9 +2,7 @@ package main
|
|||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/signal"
|
||||
"strconv"
|
||||
|
|
@ -13,11 +11,11 @@ import (
|
|||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
"github.com/rss2/backend/internal/logger"
|
||||
"github.com/rss2/backend/internal/workers"
|
||||
)
|
||||
|
||||
var (
|
||||
logger *log.Logger
|
||||
dbPool *pgxpool.Pool
|
||||
sleepSec = 10
|
||||
batchSz = 500
|
||||
|
|
@ -36,8 +34,7 @@ type Country struct {
|
|||
}
|
||||
|
||||
func init() {
|
||||
logLevel := os.Getenv("LOG_LEVEL")
|
||||
logger.Init("topics-worker", logLevel)
|
||||
logger = log.New(os.Stdout, "[TOPICS] ", log.LstdFlags)
|
||||
}
|
||||
|
||||
func loadConfig() {
|
||||
|
|
@ -279,46 +276,29 @@ func processBatch(ctx context.Context, topics []Topic, countries []Country) (int
|
|||
processedIDs = append(processedIDs, item.ID)
|
||||
}
|
||||
|
||||
// Insert topic relations (batched, upsert)
|
||||
// Insert topic relations
|
||||
if len(topicMatches) > 0 {
|
||||
const chunkSize = 500
|
||||
for i := 0; i < len(topicMatches); i += chunkSize {
|
||||
end := i + chunkSize
|
||||
if end > len(topicMatches) {
|
||||
end = len(topicMatches)
|
||||
}
|
||||
chunk := topicMatches[i:end]
|
||||
values := make([]string, 0, len(chunk))
|
||||
args := make([]interface{}, 0, len(chunk)*3)
|
||||
for j, tm := range chunk {
|
||||
values = append(values, fmt.Sprintf("($%d, $%d, $%d)", j*3+1, j*3+2, j*3+3))
|
||||
args = append(args, tm.NoticiaID, tm.TopicID, tm.Score)
|
||||
}
|
||||
query := fmt.Sprintf(`
|
||||
for _, tm := range topicMatches {
|
||||
_, err := dbPool.Exec(ctx, `
|
||||
INSERT INTO news_topics (noticia_id, topic_id, score)
|
||||
VALUES %s
|
||||
VALUES ($1, $2, $3)
|
||||
ON CONFLICT (noticia_id, topic_id) DO UPDATE SET score = EXCLUDED.score
|
||||
`, strings.Join(values, ","))
|
||||
if _, err := dbPool.Exec(ctx, query, args...); err != nil {
|
||||
logger.Printf("Error batch inserting topics: %v", err)
|
||||
`, tm.NoticiaID, tm.TopicID, tm.Score)
|
||||
if err != nil {
|
||||
logger.Printf("Error inserting topic: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Update country (single bulk UPDATE)
|
||||
// Update country
|
||||
if len(countryUpdates) > 0 {
|
||||
paisSlice := make([]int32, len(countryUpdates))
|
||||
idSlice := make([]string, len(countryUpdates))
|
||||
for i, cu := range countryUpdates {
|
||||
paisSlice[i] = int32(cu.PaisID)
|
||||
idSlice[i] = cu.NoticiaID
|
||||
}
|
||||
if _, err := dbPool.Exec(ctx, `
|
||||
UPDATE noticias n SET pais_id = u.pais
|
||||
FROM unnest($1::int[], $2::varchar[]) AS u(pais, noticia_id)
|
||||
WHERE n.id = u.noticia_id
|
||||
`, paisSlice, idSlice); err != nil {
|
||||
logger.Printf("Error bulk updating countries: %v", err)
|
||||
for _, cu := range countryUpdates {
|
||||
_, err := dbPool.Exec(ctx, `
|
||||
UPDATE noticias SET pais_id = $1 WHERE id = $2
|
||||
`, cu.PaisID, cu.NoticiaID)
|
||||
if err != nil {
|
||||
logger.Printf("Error updating country: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -337,36 +317,20 @@ func processBatch(ctx context.Context, topics []Topic, countries []Country) (int
|
|||
|
||||
func main() {
|
||||
loadConfig()
|
||||
logger.Info().Msg("Starting Topics Worker")
|
||||
logger.Println("Starting Topics Worker")
|
||||
|
||||
cfg := workers.LoadDBConfig()
|
||||
if err := workers.Connect(cfg); err != nil {
|
||||
logger.Fatal().Err(err).Msg("Failed to connect to database")
|
||||
logger.Fatalf("Failed to connect to database: %v", err)
|
||||
}
|
||||
dbPool = workers.GetPool()
|
||||
defer workers.Close()
|
||||
|
||||
// Start health check HTTP server
|
||||
go func() {
|
||||
http.HandleFunc("/health", func(w http.ResponseWriter, r *http.Request) {
|
||||
if err := workers.HealthCheck(r.Context()); err != nil {
|
||||
http.Error(w, "unhealthy", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusOK)
|
||||
w.Write([]byte("ok"))
|
||||
})
|
||||
logger.Info().Msg("Health check server listening on :8080")
|
||||
if err := http.ListenAndServe(":8080", nil); err != nil {
|
||||
logger.Error().Err(err).Msg("Health check server error")
|
||||
}
|
||||
}()
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
ctx := context.Background()
|
||||
|
||||
// Ensure schema
|
||||
if err := ensureSchema(ctx); err != nil {
|
||||
logger.Error().Err(err).Msg("Error ensuring schema")
|
||||
logger.Printf("Error ensuring schema: %v", err)
|
||||
}
|
||||
|
||||
sigChan := make(chan os.Signal, 1)
|
||||
|
|
@ -374,47 +338,41 @@ func main() {
|
|||
|
||||
go func() {
|
||||
<-sigChan
|
||||
logger.Info().Msg("Shutting down gracefully...")
|
||||
cancel()
|
||||
time.Sleep(2 * time.Second)
|
||||
workers.Close()
|
||||
logger.Println("Shutting down...")
|
||||
os.Exit(0)
|
||||
}()
|
||||
|
||||
logger.Info().Msgf("Config: sleep=%ds, batch=%d", sleepSec, batchSz)
|
||||
logger.Printf("Config: sleep=%ds, batch=%d", sleepSec, batchSz)
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
logger.Info().Msg("Context cancelled, stopping worker")
|
||||
return
|
||||
case <-time.After(time.Duration(sleepSec) * time.Second):
|
||||
topics, err := loadTopics(ctx)
|
||||
if err != nil {
|
||||
logger.Error().Err(err).Msg("Error loading topics")
|
||||
logger.Printf("Error loading topics: %v", err)
|
||||
continue
|
||||
}
|
||||
|
||||
if len(topics) == 0 {
|
||||
logger.Info().Msg("No topics found in DB")
|
||||
logger.Println("No topics found in DB")
|
||||
time.Sleep(time.Duration(sleepSec) * time.Second)
|
||||
continue
|
||||
}
|
||||
|
||||
countries, err := loadCountries(ctx)
|
||||
if err != nil {
|
||||
logger.Error().Err(err).Msg("Error loading countries")
|
||||
logger.Printf("Error loading countries: %v", err)
|
||||
continue
|
||||
}
|
||||
|
||||
count, err := processBatch(ctx, topics, countries)
|
||||
if err != nil {
|
||||
logger.Error().Err(err).Msg("Error processing batch")
|
||||
logger.Printf("Error processing batch: %v", err)
|
||||
continue
|
||||
}
|
||||
|
||||
if count > 0 {
|
||||
logger.Info().Msgf("Processed %d news items", count)
|
||||
logger.Printf("Processed %d news items", count)
|
||||
}
|
||||
|
||||
if count < batchSz {
|
||||
|
|
|
|||
|
|
@ -16,16 +16,15 @@ import (
|
|||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
"github.com/rss2/backend/internal/logger"
|
||||
"github.com/rss2/backend/internal/workers"
|
||||
)
|
||||
|
||||
var (
|
||||
logger *log.Logger
|
||||
pool *pgxpool.Pool
|
||||
sleepInterval = 30
|
||||
batchSize = 20
|
||||
imagesDir = "/app/data/wiki_images"
|
||||
maxImageBytes = int64(2 << 20)
|
||||
batchSize = 50
|
||||
imagesDir = "/opt/rss2/data/wiki_images"
|
||||
)
|
||||
|
||||
type WikiSummary struct {
|
||||
|
|
@ -52,8 +51,7 @@ type Tag struct {
|
|||
}
|
||||
|
||||
func init() {
|
||||
logLevel := os.Getenv("LOG_LEVEL")
|
||||
logger.Init("wiki-worker", logLevel)
|
||||
logger = log.New(os.Stdout, "[WIKI_WORKER] ", log.LstdFlags)
|
||||
}
|
||||
|
||||
func getPendingTags(ctx context.Context) ([]Tag, error) {
|
||||
|
|
@ -103,17 +101,13 @@ func downloadImage(imgURL, destPath string) error {
|
|||
return fmt.Errorf("HTTP %d", resp.StatusCode)
|
||||
}
|
||||
|
||||
if resp.ContentLength > maxImageBytes {
|
||||
return fmt.Errorf("image too large (%d bytes)", resp.ContentLength)
|
||||
}
|
||||
|
||||
out, err := os.Create(destPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer out.Close()
|
||||
|
||||
_, err = io.Copy(out, io.LimitReader(resp.Body, maxImageBytes))
|
||||
_, err = io.Copy(out, resp.Body)
|
||||
return err
|
||||
}
|
||||
|
||||
|
|
@ -216,101 +210,61 @@ func processTag(ctx context.Context, tag Tag) {
|
|||
}
|
||||
|
||||
func main() {
|
||||
var sleepVal, batchVal int
|
||||
if val := os.Getenv("WIKI_SLEEP"); val != "" {
|
||||
if _, err := fmt.Sscanf(val, "%d", &sleepVal); err == nil && sleepVal > 0 {
|
||||
sleepInterval = sleepVal
|
||||
}
|
||||
if val := os.Getenv("WIKI_IMAGES_PATH"); val != "" {
|
||||
imagesDir = val
|
||||
}
|
||||
if val := os.Getenv("WIKI_BATCH"); val != "" {
|
||||
if _, err := fmt.Sscanf(val, "%d", &batchVal); err == nil && batchVal > 0 {
|
||||
batchSize = batchVal
|
||||
if val := os.Getenv("WIKI_SLEEP"); val != "" {
|
||||
if sleep, err := fmt.Sscanf(val, "%d", &sleepInterval); err == nil && sleep > 0 {
|
||||
sleepInterval = sleep
|
||||
}
|
||||
}
|
||||
|
||||
logger.Info().Msg("Iniciando Wiki Worker...")
|
||||
logger.Println("Iniciando Wiki Worker...")
|
||||
|
||||
if err := os.MkdirAll(imagesDir, 0755); err != nil {
|
||||
logger.Fatal().Err(err).Msg("Error creando directorio de imágenes")
|
||||
logger.Fatalf("Error creando directorio de imágenes: %v", err)
|
||||
}
|
||||
|
||||
cfg := workers.LoadDBConfig()
|
||||
if err := workers.Connect(cfg); err != nil {
|
||||
logger.Fatal().Err(err).Msg("Failed to connect to database")
|
||||
logger.Fatalf("Failed to connect to database: %v", err)
|
||||
}
|
||||
pool = workers.GetPool()
|
||||
defer workers.Close()
|
||||
|
||||
// Start health check HTTP server
|
||||
go func() {
|
||||
http.HandleFunc("/health", func(w http.ResponseWriter, r *http.Request) {
|
||||
if err := workers.HealthCheck(r.Context()); err != nil {
|
||||
http.Error(w, "unhealthy", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusOK)
|
||||
w.Write([]byte("ok"))
|
||||
})
|
||||
logger.Info().Msg("Health check server listening on :8080")
|
||||
if err := http.ListenAndServe(":8080", nil); err != nil {
|
||||
logger.Error().Err(err).Msg("Health check server error")
|
||||
}
|
||||
}()
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
ctx := context.Background()
|
||||
|
||||
sigChan := make(chan os.Signal, 1)
|
||||
signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM)
|
||||
|
||||
go func() {
|
||||
<-sigChan
|
||||
logger.Info().Msg("Shutting down gracefully...")
|
||||
cancel()
|
||||
// Give time for in-flight requests to complete
|
||||
time.Sleep(2 * time.Second)
|
||||
logger.Println("Cerrando gracefully...")
|
||||
workers.Close()
|
||||
os.Exit(0)
|
||||
}()
|
||||
|
||||
logger.Info().Msgf("Configuración: sleep=%ds, batch=%d", sleepInterval, batchSize)
|
||||
logger.Printf("Configuración: sleep=%ds, batch=%d", sleepInterval, batchSize)
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
logger.Info().Msg("Context cancelled, stopping worker")
|
||||
return
|
||||
default:
|
||||
tags, err := getPendingTags(ctx)
|
||||
if err != nil {
|
||||
logger.Error().Err(err).Msg("Error recuperando tags pendientes")
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-time.After(10 * time.Second):
|
||||
}
|
||||
continue
|
||||
}
|
||||
tags, err := getPendingTags(ctx)
|
||||
if err != nil {
|
||||
logger.Printf("Error recuperando tags pendientes: %v", err)
|
||||
time.Sleep(10 * time.Second)
|
||||
continue
|
||||
}
|
||||
|
||||
if len(tags) == 0 {
|
||||
logger.Info().Msgf("No hay tags pendientes. Durmiendo %d segundos...", sleepInterval)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-time.After(time.Duration(sleepInterval) * time.Second):
|
||||
}
|
||||
continue
|
||||
}
|
||||
if len(tags) == 0 {
|
||||
logger.Printf("No hay tags pendientes. Durmiendo %d segundos...", sleepInterval)
|
||||
time.Sleep(time.Duration(sleepInterval) * time.Second)
|
||||
continue
|
||||
}
|
||||
|
||||
logger.Info().Msgf("Recuperados %d tags para procesar...", len(tags))
|
||||
logger.Printf("Recuperados %d tags para procesar...", len(tags))
|
||||
|
||||
for _, tag := range tags {
|
||||
processTag(ctx, tag)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-time.After(3 * time.Second): // Increased delay to avoid Wikipedia Rate Limits (429)
|
||||
}
|
||||
}
|
||||
for _, tag := range tags {
|
||||
processTag(ctx, tag)
|
||||
time.Sleep(3 * time.Second) // Increased delay to avoid Wikipedia Rate Limits (429)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,43 +0,0 @@
|
|||
// Package docs provides Swagger documentation for the RSS2 API
|
||||
//
|
||||
// @title RSS2 API
|
||||
// @version 1.0
|
||||
// @description RSS2 News Aggregator API - A comprehensive news aggregation and analysis platform
|
||||
// @termsOfService http://swagger.io/terms/
|
||||
//
|
||||
// @contact.name RSS2 Team
|
||||
// @contact.url http://github.com/rss2
|
||||
// @contact.email support@rss2.example.com
|
||||
//
|
||||
// @license.name MIT
|
||||
// @license.url https://opensource.org/licenses/MIT
|
||||
//
|
||||
// @host localhost:8080
|
||||
// @BasePath /api
|
||||
//
|
||||
// @securityDefinitions.apikey BearerAuth
|
||||
// @in header
|
||||
// @name Authorization
|
||||
// @description Type "Bearer" followed by a space and JWT token.
|
||||
//
|
||||
// @tag.name auth
|
||||
// @tag.description Authentication endpoints
|
||||
//
|
||||
// @tag.name news
|
||||
// @tag.description News management endpoints
|
||||
//
|
||||
// @tag.name feeds
|
||||
// @tag.description Feed management endpoints
|
||||
//
|
||||
// @tag.name search
|
||||
// @tag.description Search and discovery endpoints
|
||||
//
|
||||
// @tag.name entities
|
||||
// @tag.description Entity extraction and analysis
|
||||
//
|
||||
// @tag.name admin
|
||||
// @tag.description Admin management endpoints
|
||||
//
|
||||
// @tag.name stats
|
||||
// @tag.description Statistics and analytics
|
||||
package docs
|
||||
|
|
@ -1,29 +1,19 @@
|
|||
module github.com/rss2/backend
|
||||
|
||||
go 1.23
|
||||
|
||||
toolchain go1.23.12
|
||||
go 1.22
|
||||
|
||||
require (
|
||||
github.com/PuerkitoBio/goquery v1.9.2
|
||||
github.com/gin-gonic/gin v1.9.1
|
||||
github.com/golang-jwt/jwt/v5 v5.0.0
|
||||
github.com/google/uuid v1.6.0
|
||||
github.com/gorilla/websocket v1.5.1
|
||||
github.com/jackc/pgx/v5 v5.4.3
|
||||
github.com/mmcdole/gofeed v1.2.1
|
||||
github.com/redis/go-redis/v9 v9.0.5
|
||||
github.com/rs/zerolog v1.32.0
|
||||
github.com/swaggo/files v1.0.1
|
||||
github.com/swaggo/gin-swagger v1.6.0
|
||||
golang.org/x/crypto v0.28.0
|
||||
golang.org/x/time v0.5.0
|
||||
golang.org/x/crypto v0.26.0
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/KyleBanks/depth v1.2.1 // indirect
|
||||
github.com/PuerkitoBio/purell v1.1.1 // indirect
|
||||
github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect
|
||||
github.com/andybalholm/cascadia v1.3.2 // indirect
|
||||
github.com/bytedance/sonic v1.9.1 // indirect
|
||||
github.com/cespare/xxhash/v2 v2.3.0 // indirect
|
||||
|
|
@ -31,39 +21,31 @@ require (
|
|||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
|
||||
github.com/gabriel-vasile/mimetype v1.4.2 // indirect
|
||||
github.com/gin-contrib/sse v0.1.0 // indirect
|
||||
github.com/go-openapi/jsonpointer v0.19.5 // indirect
|
||||
github.com/go-openapi/jsonreference v0.19.6 // indirect
|
||||
github.com/go-openapi/spec v0.20.4 // indirect
|
||||
github.com/go-openapi/swag v0.19.15 // indirect
|
||||
github.com/go-playground/locales v0.14.1 // indirect
|
||||
github.com/go-playground/universal-translator v0.18.1 // indirect
|
||||
github.com/go-playground/validator/v10 v10.14.0 // indirect
|
||||
github.com/goccy/go-json v0.10.2 // indirect
|
||||
github.com/google/go-cmp v0.6.0 // indirect
|
||||
github.com/jackc/pgpassfile v1.0.0 // indirect
|
||||
github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect
|
||||
github.com/jackc/puddle/v2 v2.2.1 // indirect
|
||||
github.com/josharian/intern v1.0.0 // indirect
|
||||
github.com/json-iterator/go v1.1.12 // indirect
|
||||
github.com/klauspost/cpuid/v2 v2.2.4 // indirect
|
||||
github.com/kr/text v0.2.0 // indirect
|
||||
github.com/leodido/go-urn v1.2.4 // indirect
|
||||
github.com/mailru/easyjson v0.7.6 // indirect
|
||||
github.com/mattn/go-colorable v0.1.13 // indirect
|
||||
github.com/mattn/go-isatty v0.0.19 // indirect
|
||||
github.com/mmcdole/goxpp v1.1.0 // indirect
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
|
||||
github.com/modern-go/reflect2 v1.0.2 // indirect
|
||||
github.com/pelletier/go-toml/v2 v2.0.8 // indirect
|
||||
github.com/rogpeppe/go-internal v1.14.1 // indirect
|
||||
github.com/swaggo/swag v1.16.3 // indirect
|
||||
github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
|
||||
github.com/ugorji/go/codec v1.2.11 // indirect
|
||||
golang.org/x/arch v0.3.0 // indirect
|
||||
golang.org/x/net v0.30.0 // indirect
|
||||
golang.org/x/net v0.28.0 // indirect
|
||||
golang.org/x/sync v0.8.0 // indirect
|
||||
golang.org/x/sys v0.26.0 // indirect
|
||||
golang.org/x/text v0.19.0 // indirect
|
||||
golang.org/x/tools v0.26.0 // indirect
|
||||
golang.org/x/text v0.17.0 // indirect
|
||||
google.golang.org/protobuf v1.34.2 // indirect
|
||||
gopkg.in/yaml.v2 v2.4.0 // indirect
|
||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||
)
|
||||
|
|
|
|||
|
|
@ -1,11 +1,5 @@
|
|||
github.com/KyleBanks/depth v1.2.1 h1:5h8fQADFrWtarTdtDudMmGsC7GPbOAu6RVB3ffsVFHc=
|
||||
github.com/KyleBanks/depth v1.2.1/go.mod h1:jzSb9d0L43HxTQfT+oSA1EEp2q+ne2uh6XgeJcm8brE=
|
||||
github.com/PuerkitoBio/goquery v1.9.2 h1:4/wZksC3KgkQw7SQgkKotmKljk0M6V8TUvA8Wb4yPeE=
|
||||
github.com/PuerkitoBio/goquery v1.9.2/go.mod h1:GHPCaP0ODyyxqcNoFGYlAprUFH81NuRPd0GX3Zu2Mvk=
|
||||
github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI=
|
||||
github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
|
||||
github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M=
|
||||
github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
|
||||
github.com/andybalholm/cascadia v1.3.2 h1:3Xi6Dw5lHF15JtdcmAHD3i1+T8plmv7BQ/nsViSLyss=
|
||||
github.com/andybalholm/cascadia v1.3.2/go.mod h1:7gtRlve5FxPPgIgX36uWBX58OdBsSS6lUvCFb+h7KvU=
|
||||
github.com/bsm/ginkgo/v2 v2.7.0 h1:ItPMPH90RbmZJt5GtkcNvIRuGEdwlBItdNVoyzaNQao=
|
||||
|
|
@ -20,7 +14,6 @@ github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XL
|
|||
github.com/chenzhuoyu/base64x v0.0.0-20211019084208-fb5309c8db06/go.mod h1:DH46F32mSOjUmXrMHnKwZdA8wcEefY7UVqBKYGjpdQY=
|
||||
github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 h1:qSGYFH7+jGhDF8vLC+iwCD4WpbV1EBDSzWkJODFLams=
|
||||
github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311/go.mod h1:b583jCggY9gE99b6G5LEC39OIiVsWj+R97kbl5odCEk=
|
||||
github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
|
||||
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
|
|
@ -29,22 +22,10 @@ github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/r
|
|||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
|
||||
github.com/gabriel-vasile/mimetype v1.4.2 h1:w5qFW6JKBz9Y393Y4q372O9A7cUSequkh1Q7OhCmWKU=
|
||||
github.com/gabriel-vasile/mimetype v1.4.2/go.mod h1:zApsH/mKG4w07erKIaJPFiX0Tsq9BFQgN3qGY5GnNgA=
|
||||
github.com/gin-contrib/gzip v0.0.6 h1:NjcunTcGAj5CO1gn4N8jHOSIeRFHIbn51z6K+xaN4d4=
|
||||
github.com/gin-contrib/gzip v0.0.6/go.mod h1:QOJlmV2xmayAjkNS2Y8NQsMneuRShOU/kjovCXNuzzk=
|
||||
github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE=
|
||||
github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI=
|
||||
github.com/gin-gonic/gin v1.9.1 h1:4idEAncQnU5cB7BeOkPtxjfCSye0AAm1R0RVIqJ+Jmg=
|
||||
github.com/gin-gonic/gin v1.9.1/go.mod h1:hPrL7YrpYKXt5YId3A/Tnip5kqbEAP+KLuI3SUcPTeU=
|
||||
github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
|
||||
github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY=
|
||||
github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
|
||||
github.com/go-openapi/jsonreference v0.19.6 h1:UBIxjkht+AWIgYzCDSv2GN+E/togfwXUJFRTWhl2Jjs=
|
||||
github.com/go-openapi/jsonreference v0.19.6/go.mod h1:diGHMEHg2IqXZGKxqyvWdfWU/aim5Dprw5bqpKkTvns=
|
||||
github.com/go-openapi/spec v0.20.4 h1:O8hJrt0UMnhHcluhIdUgCLRWyM2x7QkBXRvOs7m+O1M=
|
||||
github.com/go-openapi/spec v0.20.4/go.mod h1:faYFR1CvsJZ0mNsmsphTMSoRrNV3TEDoAM7FOEWeq8I=
|
||||
github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
|
||||
github.com/go-openapi/swag v0.19.15 h1:D2NRCBzS9/pEY3gP9Nl8aDqGUcPFrwG2p+CNFrLyrCM=
|
||||
github.com/go-openapi/swag v0.19.15/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ=
|
||||
github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s=
|
||||
github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
|
||||
github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA=
|
||||
|
|
@ -55,7 +36,6 @@ github.com/go-playground/validator/v10 v10.14.0 h1:vgvQWe3XCz3gIeFDm/HnTIbj6UGmg
|
|||
github.com/go-playground/validator/v10 v10.14.0/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU=
|
||||
github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU=
|
||||
github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
|
||||
github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
|
||||
github.com/golang-jwt/jwt/v5 v5.0.0 h1:1n1XNM9hk7O9mnQoNBGolZvzebBQ7p93ULHRc28XJUE=
|
||||
github.com/golang-jwt/jwt/v5 v5.0.0/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk=
|
||||
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
|
||||
|
|
@ -63,8 +43,6 @@ github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeN
|
|||
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
|
||||
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/gorilla/websocket v1.5.1 h1:gmztn0JnHVt9JZquRuzLw3g4wouNVzKL15iLr/zn/QY=
|
||||
github.com/gorilla/websocket v1.5.1/go.mod h1:x3kM2JMyaluk02fnUJpQuwD2dCS5NDG2ZHL0uE0tcaY=
|
||||
github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
|
||||
github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
|
||||
github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk=
|
||||
|
|
@ -73,29 +51,17 @@ github.com/jackc/pgx/v5 v5.4.3 h1:cxFyXhxlvAifxnkKKdlxv8XqUf59tDlYjnV5YYfsJJY=
|
|||
github.com/jackc/pgx/v5 v5.4.3/go.mod h1:Ig06C2Vu0t5qXC60W8sqIthScaEnFvojjj9dSljmHRA=
|
||||
github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk=
|
||||
github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
|
||||
github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
|
||||
github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
|
||||
github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
|
||||
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
|
||||
github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
|
||||
github.com/klauspost/cpuid/v2 v2.2.4 h1:acbojRNwl3o09bUq+yDCtZFc1aiwaAAxtcn8YkZXnvk=
|
||||
github.com/klauspost/cpuid/v2 v2.2.4/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY=
|
||||
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
|
||||
github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0=
|
||||
github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk=
|
||||
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
|
||||
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
|
||||
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
|
||||
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
|
||||
github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q=
|
||||
github.com/leodido/go-urn v1.2.4/go.mod h1:7ZrI8mTSeBSHl/UaRyKQW1qZeMgak41ANeCNaVckg+4=
|
||||
github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
|
||||
github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
|
||||
github.com/mailru/easyjson v0.7.6 h1:8yTIVnZgCoiM1TgqoeTl+LfU5Jg6/xL3QhGQnimLYnA=
|
||||
github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
|
||||
github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
|
||||
github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
|
||||
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
|
||||
github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA=
|
||||
github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
|
||||
github.com/mmcdole/gofeed v1.2.1 h1:tPbFN+mfOLcM1kDF1x2c/N68ChbdBatkppdzf/vDe1s=
|
||||
|
|
@ -107,24 +73,18 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w
|
|||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
|
||||
github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
|
||||
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
|
||||
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
|
||||
github.com/pelletier/go-toml/v2 v2.0.8 h1:0ctb6s9mE31h0/lhu+J6OPmVeDxJn+kYnJc2jZR9tGQ=
|
||||
github.com/pelletier/go-toml/v2 v2.0.8/go.mod h1:vuYfssBdrU2XDZ9bYydBu6t+6a6PYNcZljzZR9VXg+4=
|
||||
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/redis/go-redis/v9 v9.0.5 h1:CuQcn5HIEeK7BgElubPP8CGtE0KakrnbBSTLjathl5o=
|
||||
github.com/redis/go-redis/v9 v9.0.5/go.mod h1:WqMKv5vnQbRuZstUwxQI195wHy+t4PuXDOjzMvcuQHk=
|
||||
github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
|
||||
github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
|
||||
github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
|
||||
github.com/rs/zerolog v1.32.0 h1:keLypqrlIjaFsbmJOBdB/qvyF8KEtCWHwobLp5l/mQ0=
|
||||
github.com/rs/zerolog v1.32.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
|
||||
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
|
||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
|
||||
|
|
@ -132,12 +92,6 @@ github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o
|
|||
github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
|
||||
github.com/stretchr/testify v1.8.3 h1:RP3t2pwF7cMEbC1dqtB6poj3niw/9gnV4Cjg5oW5gtY=
|
||||
github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
|
||||
github.com/swaggo/files v1.0.1 h1:J1bVJ4XHZNq0I46UU90611i9/YzdrF7x92oX1ig5IdE=
|
||||
github.com/swaggo/files v1.0.1/go.mod h1:0qXmMNH6sXNf+73t65aKeB+ApmgxdnkQzVTAj2uaMUg=
|
||||
github.com/swaggo/gin-swagger v1.6.0 h1:y8sxvQ3E20/RCyrXeFfg60r6H0Z+SwpTjMYsMm+zy8M=
|
||||
github.com/swaggo/gin-swagger v1.6.0/go.mod h1:BG00cCEy294xtVpyIAHG6+e2Qzj/xKlRdOqDkvq0uzo=
|
||||
github.com/swaggo/swag v1.16.3 h1:PnCYjPCah8FK4I26l2F/KQ4yz3sILcVUN3cTlBFA9Pg=
|
||||
github.com/swaggo/swag v1.16.3/go.mod h1:DImHIuOFXKpMFAQjcC7FG4m3Dg4+QuUgUzJmKjI/gRk=
|
||||
github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI=
|
||||
github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08=
|
||||
github.com/ugorji/go/codec v1.2.11 h1:BMaWp1Bb6fHwEtbplGBGJ498wD+LKlNSl25MjdZY4dU=
|
||||
|
|
@ -148,21 +102,17 @@ golang.org/x/arch v0.3.0 h1:02VY4/ZcO/gBOH6PUaoiptASxtXU10jazRCP865E97k=
|
|||
golang.org/x/arch v0.3.0/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
|
||||
golang.org/x/crypto v0.28.0 h1:GBDwsMXVQi34v5CCYUm2jkJvu4cbtru2U4TN2PSyQnw=
|
||||
golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U=
|
||||
golang.org/x/crypto v0.26.0 h1:RrRspgV4mU+YwB4FYnuBoKsUapNIL5cohGAmSH3azsw=
|
||||
golang.org/x/crypto v0.26.0/go.mod h1:GY7jblb9wI+FOo5y8/S2oY4zWP07AkOJ4+jxCqdqn54=
|
||||
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
|
||||
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
|
||||
golang.org/x/mod v0.21.0 h1:vvrHzRwRfVKSiLrG+d4FMl/Qi4ukBCE6kZlTUkDYRT0=
|
||||
golang.org/x/mod v0.21.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY=
|
||||
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
|
||||
golang.org/x/net v0.0.0-20210421230115-4e50805a0758/go.mod h1:72T/g9IO56b78aLF+1Kcs5dz7/ng1VjMUvfKvpfy+jM=
|
||||
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
|
||||
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
|
||||
golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
|
||||
golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns=
|
||||
golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4=
|
||||
golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU=
|
||||
golang.org/x/net v0.28.0 h1:a9JDOJc5GMUJ0+UDqmLT86WiEy7iWyIhz8gz8E4e5hE=
|
||||
golang.org/x/net v0.28.0/go.mod h1:yqtgsTWOOnlGLG9GFRrK3++bGOUEkNBoHZc8MEDWPNg=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
|
|
@ -170,16 +120,13 @@ golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ=
|
|||
golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210420072515-93ed5bcd2bfe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo=
|
||||
golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
|
|
@ -188,33 +135,22 @@ golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
|
|||
golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
|
||||
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
|
||||
golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
|
||||
golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM=
|
||||
golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=
|
||||
golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk=
|
||||
golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
|
||||
golang.org/x/text v0.17.0 h1:XtiM5bkSOt+ewxlOE/aE/AKEHibwj/6gvWMl9Rsh0Qc=
|
||||
golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
|
||||
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
|
||||
golang.org/x/tools v0.26.0 h1:v/60pFQmzmT9ExmjDv2gGIfi3OqfKoEP6I5+umXlbnQ=
|
||||
golang.org/x/tools v0.26.0/go.mod h1:TPVVj70c7JJ3WCazhD8OdXcZg/og+b9+tH/KxylGwH0=
|
||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg=
|
||||
google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
|
||||
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
|
||||
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4=
|
||||
|
|
|
|||
|
|
@ -1,69 +0,0 @@
|
|||
package auth
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"github.com/golang-jwt/jwt/v5"
|
||||
)
|
||||
|
||||
var jwtSecret []byte
|
||||
|
||||
func SetJWTSecret(secret string) {
|
||||
jwtSecret = []byte(secret)
|
||||
}
|
||||
|
||||
func GetJWTSecret() []byte {
|
||||
return jwtSecret
|
||||
}
|
||||
|
||||
type Claims struct {
|
||||
UserID int64 `json:"user_id"`
|
||||
Email string `json:"email"`
|
||||
Username string `json:"username"`
|
||||
IsAdmin bool `json:"is_admin"`
|
||||
jwt.RegisteredClaims
|
||||
}
|
||||
|
||||
func GenerateToken(userID int64, email, username string, isAdmin bool) (string, error) {
|
||||
expirationTime := time.Now().Add(24 * time.Hour)
|
||||
claims := &Claims{
|
||||
UserID: userID,
|
||||
Email: email,
|
||||
Username: username,
|
||||
IsAdmin: isAdmin,
|
||||
RegisteredClaims: jwt.RegisteredClaims{
|
||||
ExpiresAt: jwt.NewNumericDate(expirationTime),
|
||||
IssuedAt: jwt.NewNumericDate(time.Now()),
|
||||
},
|
||||
}
|
||||
|
||||
token := jwt.NewWithClaims(jwt.SigningMethodHS256, claims)
|
||||
return token.SignedString(jwtSecret)
|
||||
}
|
||||
|
||||
func ValidateToken(tokenString string) (*Claims, error) {
|
||||
claims := &Claims{}
|
||||
token, err := jwt.ParseWithClaims(tokenString, claims, func(token *jwt.Token) (interface{}, error) {
|
||||
return jwtSecret, nil
|
||||
})
|
||||
|
||||
if err != nil || !token.Valid {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return claims, nil
|
||||
}
|
||||
|
||||
type contextKey string
|
||||
|
||||
const userContextKey contextKey = "user"
|
||||
|
||||
func SetTestUser(ctx context.Context, claims *Claims) context.Context {
|
||||
return context.WithValue(ctx, userContextKey, claims)
|
||||
}
|
||||
|
||||
func GetUserFromContext(ctx context.Context) (*Claims, bool) {
|
||||
claims, ok := ctx.Value(userContextKey).(*Claims)
|
||||
return claims, ok
|
||||
}
|
||||
|
|
@ -1,96 +0,0 @@
|
|||
package auth
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestGenerateAndValidateToken(t *testing.T) {
|
||||
SetJWTSecret("test-secret-key-12345")
|
||||
|
||||
token, err := GenerateToken(1, "test@example.com", "testuser", true)
|
||||
if err != nil {
|
||||
t.Fatalf("GenerateToken failed: %v", err)
|
||||
}
|
||||
|
||||
if token == "" {
|
||||
t.Fatal("token should not be empty")
|
||||
}
|
||||
|
||||
claims, err := ValidateToken(token)
|
||||
if err != nil {
|
||||
t.Fatalf("ValidateToken failed: %v", err)
|
||||
}
|
||||
|
||||
if claims.UserID != 1 {
|
||||
t.Errorf("expected userID 1, got %d", claims.UserID)
|
||||
}
|
||||
if claims.Email != "test@example.com" {
|
||||
t.Errorf("expected email test@example.com, got %s", claims.Email)
|
||||
}
|
||||
if claims.Username != "testuser" {
|
||||
t.Errorf("expected username testuser, got %s", claims.Username)
|
||||
}
|
||||
if !claims.IsAdmin {
|
||||
t.Error("expected isAdmin true")
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateInvalidToken(t *testing.T) {
|
||||
SetJWTSecret("test-secret-key-12345")
|
||||
|
||||
_, err := ValidateToken("invalid-token")
|
||||
if err == nil {
|
||||
t.Error("expected error for invalid token")
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateTokenWithWrongSecret(t *testing.T) {
|
||||
SetJWTSecret("secret-1")
|
||||
|
||||
token, err := GenerateToken(1, "test@example.com", "testuser", false)
|
||||
if err != nil {
|
||||
t.Fatalf("GenerateToken failed: %v", err)
|
||||
}
|
||||
|
||||
SetJWTSecret("secret-2")
|
||||
|
||||
_, err = ValidateToken(token)
|
||||
if err == nil {
|
||||
t.Error("expected error for token with different secret")
|
||||
}
|
||||
}
|
||||
|
||||
func TestClaimsHaveExpiration(t *testing.T) {
|
||||
SetJWTSecret("test-secret-key-12345")
|
||||
|
||||
token, err := GenerateToken(1, "test@example.com", "testuser", false)
|
||||
if err != nil {
|
||||
t.Fatalf("GenerateToken failed: %v", err)
|
||||
}
|
||||
|
||||
claims, err := ValidateToken(token)
|
||||
if err != nil {
|
||||
t.Fatalf("ValidateToken failed: %v", err)
|
||||
}
|
||||
|
||||
if claims.ExpiresAt == nil {
|
||||
t.Error("expected expiration time to be set")
|
||||
}
|
||||
|
||||
expTime := claims.ExpiresAt.Time
|
||||
expectedExpiration := time.Now().Add(24 * time.Hour)
|
||||
|
||||
if expTime.Sub(expectedExpiration) > time.Minute {
|
||||
t.Errorf("expiration time should be ~24 hours from now, got %v", expTime)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEmptyToken(t *testing.T) {
|
||||
SetJWTSecret("test-secret-key-12345")
|
||||
|
||||
_, err := ValidateToken("")
|
||||
if err == nil {
|
||||
t.Error("expected error for empty token")
|
||||
}
|
||||
}
|
||||
111
backend/internal/cache/redis.go
vendored
111
backend/internal/cache/redis.go
vendored
|
|
@ -11,26 +11,6 @@ import (
|
|||
|
||||
var Client *redis.Client
|
||||
|
||||
// Cache TTL constants
|
||||
const (
|
||||
TTLShort = 5 * time.Minute // 5 min for frequently changing data
|
||||
TTLMedium = 30 * time.Minute // 30 min for semi-static data
|
||||
TTLLong = 2 * time.Hour // 2 hours for stable data
|
||||
TTLNoExpire = 0 // No expiration (manual invalidation)
|
||||
)
|
||||
|
||||
const (
|
||||
KeyPrefixSearch = "search"
|
||||
KeyPrefixNews = "news"
|
||||
KeyPrefixFeed = "feed"
|
||||
KeyPrefixFeedList = "feedlist"
|
||||
KeyPrefixCategories = "categories"
|
||||
KeyPrefixCountries = "countries"
|
||||
KeyPrefixStats = "stats"
|
||||
KeyPrefixEntities = "entities"
|
||||
KeyPrefixStatsTop = "statstop"
|
||||
)
|
||||
|
||||
func Connect(redisURL string) error {
|
||||
opt, err := redis.ParseURL(redisURL)
|
||||
if err != nil {
|
||||
|
|
@ -60,39 +40,15 @@ func GetClient() *redis.Client {
|
|||
}
|
||||
|
||||
func SearchKey(query, lang string, page, perPage int) string {
|
||||
return fmt.Sprintf("%s:%s:%s:%d:%d", KeyPrefixSearch, query, lang, page, perPage)
|
||||
return fmt.Sprintf("search:%s:%s:%d:%d", query, lang, page, perPage)
|
||||
}
|
||||
|
||||
func NewsKey(newsID int64, lang string) string {
|
||||
return fmt.Sprintf("%s:%d:%s", KeyPrefixNews, newsID, lang)
|
||||
return fmt.Sprintf("news:%d:%s", newsID, lang)
|
||||
}
|
||||
|
||||
func FeedKey(feedID int64) string {
|
||||
return fmt.Sprintf("%s:%d", KeyPrefixFeed, feedID)
|
||||
}
|
||||
|
||||
func FeedListKey(filters string, page, perPage int) string {
|
||||
return fmt.Sprintf("%s:%s:%d:%d", KeyPrefixFeedList, filters, page, perPage)
|
||||
}
|
||||
|
||||
func CategoriesKey() string {
|
||||
return KeyPrefixCategories
|
||||
}
|
||||
|
||||
func CountriesKey() string {
|
||||
return KeyPrefixCountries
|
||||
}
|
||||
|
||||
func StatsKey() string {
|
||||
return KeyPrefixStats
|
||||
}
|
||||
|
||||
func StatsTopKey(typ string) string {
|
||||
return fmt.Sprintf("%s:%s", KeyPrefixStatsTop, typ)
|
||||
}
|
||||
|
||||
func EntityKey(entityType, query string, page, perPage int) string {
|
||||
return fmt.Sprintf("%s:%s:%s:%d:%d", KeyPrefixEntities, entityType, query, page, perPage)
|
||||
return fmt.Sprintf("feed:%d", feedID)
|
||||
}
|
||||
|
||||
func Set(ctx context.Context, key string, value interface{}, expiration time.Duration) error {
|
||||
|
|
@ -107,62 +63,6 @@ func Get(ctx context.Context, key string) (string, error) {
|
|||
return Client.Get(ctx, key).Result()
|
||||
}
|
||||
|
||||
func Delete(ctx context.Context, keys ...string) error {
|
||||
if len(keys) == 0 {
|
||||
return nil
|
||||
}
|
||||
return Client.Del(ctx, keys...).Err()
|
||||
}
|
||||
|
||||
func DeletePattern(ctx context.Context, pattern string) error {
|
||||
iter := Client.Scan(ctx, 0, pattern, 100).Iterator()
|
||||
for iter.Next(ctx) {
|
||||
if err := Client.Del(ctx, iter.Val()).Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return iter.Err()
|
||||
}
|
||||
|
||||
func InvalidateSearch(ctx context.Context) error {
|
||||
return DeletePattern(ctx, KeyPrefixSearch+":*")
|
||||
}
|
||||
|
||||
func InvalidateNews(ctx context.Context, newsID int64) error {
|
||||
pattern := fmt.Sprintf("%s:%d:*", KeyPrefixNews, newsID)
|
||||
return DeletePattern(ctx, pattern)
|
||||
}
|
||||
|
||||
func InvalidateFeed(ctx context.Context, feedID int64) error {
|
||||
return Delete(ctx, FeedKey(feedID))
|
||||
}
|
||||
|
||||
func InvalidateFeedList(ctx context.Context) error {
|
||||
return DeletePattern(ctx, KeyPrefixFeedList+":*")
|
||||
}
|
||||
|
||||
func InvalidateCategories(ctx context.Context) error {
|
||||
return Delete(ctx, CategoriesKey())
|
||||
}
|
||||
|
||||
func InvalidateCountries(ctx context.Context) error {
|
||||
return Delete(ctx, CountriesKey())
|
||||
}
|
||||
|
||||
func InvalidateStats(ctx context.Context) error {
|
||||
keys := []string{
|
||||
StatsKey(),
|
||||
StatsTopKey("categories"),
|
||||
StatsTopKey("countries"),
|
||||
}
|
||||
return Delete(ctx, keys...)
|
||||
}
|
||||
|
||||
func InvalidateEntities(ctx context.Context, entityType string) error {
|
||||
pattern := fmt.Sprintf("%s:%s:*", KeyPrefixEntities, entityType)
|
||||
return DeletePattern(ctx, pattern)
|
||||
}
|
||||
|
||||
func Unmarshal(data []byte, v interface{}) error {
|
||||
return json.Unmarshal(data, v)
|
||||
}
|
||||
|
|
@ -170,8 +70,3 @@ func Unmarshal(data []byte, v interface{}) error {
|
|||
func Marshal(v interface{}) ([]byte, error) {
|
||||
return json.Marshal(v)
|
||||
}
|
||||
|
||||
func Exists(ctx context.Context, key string) (bool, error) {
|
||||
count, err := Client.Exists(ctx, key).Result()
|
||||
return count > 0, err
|
||||
}
|
||||
|
|
|
|||
|
|
@ -20,39 +20,23 @@ type Config struct {
|
|||
DefaultLang string
|
||||
NewsPerPage int
|
||||
RateLimitPerMinute int
|
||||
DockerComposeDir string
|
||||
WikiImagesPath string
|
||||
AllowedOrigins string
|
||||
}
|
||||
|
||||
func Load() *Config {
|
||||
secretKey := os.Getenv("SECRET_KEY")
|
||||
if secretKey == "" {
|
||||
panic("SECRET_KEY environment variable is required")
|
||||
}
|
||||
|
||||
allowedOrigins := os.Getenv("ALLOWED_ORIGINS")
|
||||
if allowedOrigins == "" {
|
||||
allowedOrigins = "http://localhost:5173,http://localhost:3000"
|
||||
}
|
||||
|
||||
return &Config{
|
||||
ServerPort: getEnv("SERVER_PORT", "8080"),
|
||||
DatabaseURL: getEnv("DATABASE_URL", "postgres://rss:rss@localhost:5432/rss"),
|
||||
RedisURL: getEnv("REDIS_URL", "redis://localhost:6379"),
|
||||
QdrantHost: getEnv("QDRANT_HOST", "localhost"),
|
||||
QdrantPort: getEnvInt("QDRANT_PORT", 6333),
|
||||
SecretKey: secretKey,
|
||||
JWTExpiration: getEnvDuration("JWT_EXPIRATION", 24*time.Hour),
|
||||
TranslationURL: getEnv("TRANSLATION_URL", "http://libretranslate:7790"),
|
||||
OllamaURL: getEnv("OLLAMA_URL", "http://ollama:11434"),
|
||||
SpacyURL: getEnv("SPACY_URL", "http://spacy:8000"),
|
||||
DefaultLang: getEnv("DEFAULT_LANG", "es"),
|
||||
NewsPerPage: getEnvInt("NEWS_PER_PAGE", 30),
|
||||
RateLimitPerMinute: getEnvInt("RATE_LIMIT_PER_MINUTE", 60),
|
||||
DockerComposeDir: getEnv("DOCKER_COMPOSE_DIR", "/datos/rss2"),
|
||||
WikiImagesPath: getEnv("WIKI_IMAGES_PATH", "/app/data/wiki_images"),
|
||||
AllowedOrigins: allowedOrigins,
|
||||
ServerPort: getEnv("SERVER_PORT", "8080"),
|
||||
DatabaseURL: getEnv("DATABASE_URL", "postgres://rss:rss@localhost:5432/rss"),
|
||||
RedisURL: getEnv("REDIS_URL", "redis://localhost:6379"),
|
||||
QdrantHost: getEnv("QDRANT_HOST", "localhost"),
|
||||
QdrantPort: getEnvInt("QDRANT_PORT", 6333),
|
||||
SecretKey: getEnv("SECRET_KEY", "change-this-secret-key"),
|
||||
JWTExpiration: getEnvDuration("JWT_EXPIRATION", 24*time.Hour),
|
||||
TranslationURL: getEnv("TRANSLATION_URL", "http://libretranslate:7790"),
|
||||
OllamaURL: getEnv("OLLAMA_URL", "http://ollama:11434"),
|
||||
SpacyURL: getEnv("SPACY_URL", "http://spacy:8000"),
|
||||
DefaultLang: getEnv("DEFAULT_LANG", "es"),
|
||||
NewsPerPage: getEnvInt("NEWS_PER_PAGE", 30),
|
||||
RateLimitPerMinute: getEnvInt("RATE_LIMIT_PER_MINUTE", 60),
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,145 +0,0 @@
|
|||
package config
|
||||
|
||||
import (
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestLoadDefaults(t *testing.T) {
|
||||
os.Clearenv()
|
||||
|
||||
cfg := Load()
|
||||
|
||||
if cfg.ServerPort != "8080" {
|
||||
t.Errorf("expected ServerPort '8080', got %s", cfg.ServerPort)
|
||||
}
|
||||
if cfg.SecretKey != "change-this-secret-key" {
|
||||
t.Errorf("expected SecretKey 'change-this-secret-key', got %s", cfg.SecretKey)
|
||||
}
|
||||
if cfg.DefaultLang != "es" {
|
||||
t.Errorf("expected DefaultLang 'es', got %s", cfg.DefaultLang)
|
||||
}
|
||||
if cfg.NewsPerPage != 30 {
|
||||
t.Errorf("expected NewsPerPage 30, got %d", cfg.NewsPerPage)
|
||||
}
|
||||
if cfg.DockerComposeDir != "/datos/rss2" {
|
||||
t.Errorf("expected DockerComposeDir '/datos/rss2', got %s", cfg.DockerComposeDir)
|
||||
}
|
||||
if cfg.WikiImagesPath != "/app/data/wiki_images" {
|
||||
t.Errorf("expected WikiImagesPath '/app/data/wiki_images', got %s", cfg.WikiImagesPath)
|
||||
}
|
||||
if cfg.AllowedOrigins != "*" {
|
||||
t.Errorf("expected AllowedOrigins '*', got %s", cfg.AllowedOrigins)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadFromEnv(t *testing.T) {
|
||||
os.Clearenv()
|
||||
os.Setenv("SERVER_PORT", "9000")
|
||||
os.Setenv("DATABASE_URL", "postgres://user:pass@host:5432/db")
|
||||
os.Setenv("SECRET_KEY", "my-secret-key")
|
||||
os.Setenv("DEFAULT_LANG", "en")
|
||||
os.Setenv("NEWS_PER_PAGE", "50")
|
||||
os.Setenv("JWT_EXPIRATION", "48h")
|
||||
os.Setenv("RATE_LIMIT_PER_MINUTE", "100")
|
||||
os.Setenv("DOCKER_COMPOSE_DIR", "/custom/path")
|
||||
os.Setenv("WIKI_IMAGES_PATH", "/custom/images")
|
||||
|
||||
cfg := Load()
|
||||
|
||||
if cfg.ServerPort != "9000" {
|
||||
t.Errorf("expected ServerPort '9000', got %s", cfg.ServerPort)
|
||||
}
|
||||
if cfg.DatabaseURL != "postgres://user:pass@host:5432/db" {
|
||||
t.Errorf("expected DatabaseURL, got %s", cfg.DatabaseURL)
|
||||
}
|
||||
if cfg.SecretKey != "my-secret-key" {
|
||||
t.Errorf("expected SecretKey 'my-secret-key', got %s", cfg.SecretKey)
|
||||
}
|
||||
if cfg.DefaultLang != "en" {
|
||||
t.Errorf("expected DefaultLang 'en', got %s", cfg.DefaultLang)
|
||||
}
|
||||
if cfg.NewsPerPage != 50 {
|
||||
t.Errorf("expected NewsPerPage 50, got %d", cfg.NewsPerPage)
|
||||
}
|
||||
if cfg.JWTExpiration != 48*time.Hour {
|
||||
t.Errorf("expected JWTExpiration 48h, got %v", cfg.JWTExpiration)
|
||||
}
|
||||
if cfg.RateLimitPerMinute != 100 {
|
||||
t.Errorf("expected RateLimitPerMinute 100, got %d", cfg.RateLimitPerMinute)
|
||||
}
|
||||
if cfg.DockerComposeDir != "/custom/path" {
|
||||
t.Errorf("expected DockerComposeDir '/custom/path', got %s", cfg.DockerComposeDir)
|
||||
}
|
||||
if cfg.WikiImagesPath != "/custom/images" {
|
||||
t.Errorf("expected WikiImagesPath '/custom/images', got %s", cfg.WikiImagesPath)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadInvalidInt(t *testing.T) {
|
||||
os.Clearenv()
|
||||
os.Setenv("NEWS_PER_PAGE", "invalid")
|
||||
|
||||
cfg := Load()
|
||||
|
||||
if cfg.NewsPerPage != 30 {
|
||||
t.Errorf("expected default NewsPerPage 30 for invalid value, got %d", cfg.NewsPerPage)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadInvalidDuration(t *testing.T) {
|
||||
os.Clearenv()
|
||||
os.Setenv("JWT_EXPIRATION", "invalid")
|
||||
|
||||
cfg := Load()
|
||||
|
||||
if cfg.JWTExpiration != 24*time.Hour {
|
||||
t.Errorf("expected default JWTExpiration 24h for invalid value, got %v", cfg.JWTExpiration)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetEnv(t *testing.T) {
|
||||
os.Clearenv()
|
||||
os.Setenv("TEST_KEY", "test_value")
|
||||
|
||||
result := getEnv("TEST_KEY", "default")
|
||||
if result != "test_value" {
|
||||
t.Errorf("expected 'test_value', got %s", result)
|
||||
}
|
||||
|
||||
result = getEnv("NON_EXISTENT", "default")
|
||||
if result != "default" {
|
||||
t.Errorf("expected 'default', got %s", result)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetEnvInt(t *testing.T) {
|
||||
os.Clearenv()
|
||||
os.Setenv("TEST_INT", "123")
|
||||
|
||||
result := getEnvInt("TEST_INT", 0)
|
||||
if result != 123 {
|
||||
t.Errorf("expected 123, got %d", result)
|
||||
}
|
||||
|
||||
result = getEnvInt("NON_EXISTENT", 99)
|
||||
if result != 99 {
|
||||
t.Errorf("expected 99, got %d", result)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetEnvDuration(t *testing.T) {
|
||||
os.Clearenv()
|
||||
os.Setenv("TEST_DURATION", "1h30m")
|
||||
|
||||
result := getEnvDuration("TEST_DURATION", 0)
|
||||
if result != 90*time.Minute {
|
||||
t.Errorf("expected 90m, got %v", result)
|
||||
}
|
||||
|
||||
result = getEnvDuration("NON_EXISTENT", 5*time.Minute)
|
||||
if result != 5*time.Minute {
|
||||
t.Errorf("expected 5m, got %v", result)
|
||||
}
|
||||
}
|
||||
|
|
@ -3,8 +3,6 @@ package db
|
|||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
|
|
@ -18,11 +16,10 @@ func Connect(databaseURL string) error {
|
|||
return fmt.Errorf("failed to parse database URL: %w", err)
|
||||
}
|
||||
|
||||
config.MaxConns = int32(getEnvInt("DB_MAX_CONNS", 25))
|
||||
config.MinConns = int32(getEnvInt("DB_MIN_CONNS", 5))
|
||||
config.MaxConnLifetime = getEnvDuration("DB_MAX_CONN_LIFETIME", time.Hour)
|
||||
config.MaxConnIdleTime = getEnvDuration("DB_MAX_CONN_IDLE_TIME", 30*time.Minute)
|
||||
config.HealthCheckPeriod = getEnvDuration("DB_HEALTH_CHECK_PERIOD", time.Minute)
|
||||
config.MaxConns = 25
|
||||
config.MinConns = 5
|
||||
config.MaxConnLifetime = time.Hour
|
||||
config.MaxConnIdleTime = 30 * time.Minute
|
||||
|
||||
Pool, err = pgxpool.NewWithConfig(context.Background(), config)
|
||||
if err != nil {
|
||||
|
|
@ -45,28 +42,3 @@ func Close() {
|
|||
func GetPool() *pgxpool.Pool {
|
||||
return Pool
|
||||
}
|
||||
|
||||
func HealthCheck(ctx context.Context) error {
|
||||
if Pool == nil {
|
||||
return fmt.Errorf("pool not initialized")
|
||||
}
|
||||
return Pool.Ping(ctx)
|
||||
}
|
||||
|
||||
func getEnvInt(key string, defaultValue int) int {
|
||||
if value := os.Getenv(key); value != "" {
|
||||
if intVal, err := strconv.Atoi(value); err == nil {
|
||||
return intVal
|
||||
}
|
||||
}
|
||||
return defaultValue
|
||||
}
|
||||
|
||||
func getEnvDuration(key string, defaultValue time.Duration) time.Duration {
|
||||
if value := os.Getenv(key); value != "" {
|
||||
if duration, err := time.ParseDuration(value); err == nil {
|
||||
return duration
|
||||
}
|
||||
}
|
||||
return defaultValue
|
||||
}
|
||||
|
|
|
|||
|
|
@ -6,22 +6,20 @@ import (
|
|||
"context"
|
||||
"encoding/csv"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"math"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"os/exec"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/rss2/backend/internal/config"
|
||||
"github.com/rss2/backend/internal/db"
|
||||
"github.com/rss2/backend/internal/models"
|
||||
)
|
||||
|
||||
|
||||
|
||||
func CreateAlias(c *gin.Context) {
|
||||
var req models.EntityAliasRequest
|
||||
if err := c.ShouldBindJSON(&req); err != nil {
|
||||
|
|
@ -112,6 +110,8 @@ func CreateAlias(c *gin.Context) {
|
|||
})
|
||||
}
|
||||
|
||||
|
||||
|
||||
func ExportAliases(c *gin.Context) {
|
||||
rows, err := db.GetPool().Query(c.Request.Context(),
|
||||
"SELECT alias, canonical_name, tipo FROM entity_aliases ORDER BY tipo, canonical_name")
|
||||
|
|
@ -387,51 +387,6 @@ func GetWorkerStatus(c *gin.Context) {
|
|||
})
|
||||
}
|
||||
|
||||
// GetTranslationStats returns real translation throughput and live worker count.
|
||||
func GetTranslationStats(c *gin.Context) {
|
||||
ctx := c.Request.Context()
|
||||
var last1min, last5min, activeWorkers int64
|
||||
var elapsed1min, elapsed5min float64
|
||||
|
||||
db.GetPool().QueryRow(ctx, `
|
||||
SELECT COALESCE(SUM(items_translated),0), COALESCE(SUM(elapsed_ms),0)
|
||||
FROM translation_stats WHERE created_at >= NOW() - INTERVAL '1 minute'
|
||||
`).Scan(&last1min, &elapsed1min)
|
||||
|
||||
db.GetPool().QueryRow(ctx, `
|
||||
SELECT COALESCE(SUM(items_translated),0), COALESCE(SUM(elapsed_ms),0)
|
||||
FROM translation_stats WHERE created_at >= NOW() - INTERVAL '5 minutes'
|
||||
`).Scan(&last5min, &elapsed5min)
|
||||
|
||||
// Live workers = distinct hosts that recorded stats in the last 2 minutes
|
||||
db.GetPool().QueryRow(ctx, `
|
||||
SELECT COUNT(DISTINCT hostname) FROM translation_stats
|
||||
WHERE created_at >= NOW() - INTERVAL '2 minutes' AND hostname IS NOT NULL
|
||||
`).Scan(&activeWorkers)
|
||||
|
||||
// Also count WS remote workers currently online
|
||||
remoteOnline := 0
|
||||
if err := db.GetPool().QueryRow(ctx, `
|
||||
SELECT COUNT(*) FROM remote_workers WHERE status = 'online'
|
||||
`).Scan(&remoteOnline); err != nil {
|
||||
remoteOnline = 0
|
||||
}
|
||||
|
||||
rateSec := 0.0
|
||||
if elapsed1min > 0 {
|
||||
rateSec = float64(last1min) / (elapsed1min / 1000.0)
|
||||
}
|
||||
|
||||
c.JSON(http.StatusOK, gin.H{
|
||||
"translations_last_1min": last1min,
|
||||
"translations_last_5min": last5min,
|
||||
"rate_per_second": math.Round(rateSec*100) / 100,
|
||||
"rate_per_minute": math.Round(rateSec*60*100) / 100,
|
||||
"active_workers": activeWorkers + int64(remoteOnline),
|
||||
"remote_workers_online": remoteOnline,
|
||||
})
|
||||
}
|
||||
|
||||
func SetWorkerConfig(c *gin.Context) {
|
||||
var req WorkerConfig
|
||||
if err := c.ShouldBindJSON(&req); err != nil {
|
||||
|
|
@ -510,14 +465,13 @@ func StartWorkers(c *gin.Context) {
|
|||
}
|
||||
|
||||
// Detener cualquier translator existente
|
||||
composeDir := config.Load().DockerComposeDir
|
||||
stopCmd := exec.Command("docker", "compose", "stop", "translator", "translator-gpu")
|
||||
stopCmd.Dir = composeDir
|
||||
stopCmd.Dir = "/datos/rss2"
|
||||
stopCmd.Run()
|
||||
|
||||
// Iniciar con el número de workers
|
||||
startCmd := exec.Command("docker", "compose", "up", "-d", "--scale", fmt.Sprintf("%s=%d", serviceName, workers), serviceName)
|
||||
startCmd.Dir = composeDir
|
||||
startCmd.Dir = "/datos/rss2"
|
||||
output, err := startCmd.CombinedOutput()
|
||||
|
||||
if err != nil {
|
||||
|
|
@ -529,15 +483,9 @@ func StartWorkers(c *gin.Context) {
|
|||
}
|
||||
|
||||
// Actualizar estado en BD
|
||||
if _, err := db.GetPool().Exec(ctx, "UPDATE config SET value = 'running', updated_at = NOW() WHERE key = 'translator_status'"); err != nil {
|
||||
log.Printf("Failed to update translator_status: %v", err)
|
||||
}
|
||||
if _, err := db.GetPool().Exec(ctx, "UPDATE config SET value = $1, updated_at = NOW() WHERE key = 'translator_type'", translatorType); err != nil {
|
||||
log.Printf("Failed to update translator_type: %v", err)
|
||||
}
|
||||
if _, err := db.GetPool().Exec(ctx, "UPDATE config SET value = $1, updated_at = NOW() WHERE key = 'translator_workers'", translatorWorkers); err != nil {
|
||||
log.Printf("Failed to update translator_workers: %v", err)
|
||||
}
|
||||
db.GetPool().Exec(ctx, "UPDATE config SET value = 'running', updated_at = NOW() WHERE key = 'translator_status'")
|
||||
db.GetPool().Exec(ctx, "UPDATE config SET value = $1, updated_at = NOW() WHERE key = 'translator_type'", translatorType)
|
||||
db.GetPool().Exec(ctx, "UPDATE config SET value = $1, updated_at = NOW() WHERE key = 'translator_workers'", translatorWorkers)
|
||||
|
||||
c.JSON(http.StatusOK, gin.H{
|
||||
"message": "Workers started successfully",
|
||||
|
|
@ -549,9 +497,8 @@ func StartWorkers(c *gin.Context) {
|
|||
|
||||
func StopWorkers(c *gin.Context) {
|
||||
// Detener traductores
|
||||
composeDir := config.Load().DockerComposeDir
|
||||
cmd := exec.Command("docker", "compose", "stop", "translator", "translator-gpu")
|
||||
cmd.Dir = composeDir
|
||||
cmd.Dir = "/datos/rss2"
|
||||
output, err := cmd.CombinedOutput()
|
||||
|
||||
if err != nil {
|
||||
|
|
@ -563,9 +510,7 @@ func StopWorkers(c *gin.Context) {
|
|||
}
|
||||
|
||||
// Actualizar estado en BD
|
||||
if _, err := db.GetPool().Exec(c.Request.Context(), "UPDATE config SET value = 'stopped', updated_at = NOW() WHERE key = 'translator_status'"); err != nil {
|
||||
log.Printf("Failed to update translator_status: %v", err)
|
||||
}
|
||||
db.GetPool().Exec(c.Request.Context(), "UPDATE config SET value = 'stopped', updated_at = NOW() WHERE key = 'translator_status'")
|
||||
|
||||
c.JSON(http.StatusOK, gin.H{
|
||||
"message": "Workers stopped successfully",
|
||||
|
|
@ -576,8 +521,8 @@ func StopWorkers(c *gin.Context) {
|
|||
// PatchEntityTipo changes the tipo of all tags matching a given valor
|
||||
func PatchEntityTipo(c *gin.Context) {
|
||||
var req struct {
|
||||
Valor string `json:"valor" binding:"required"`
|
||||
NewTipo string `json:"new_tipo" binding:"required"`
|
||||
Valor string `json:"valor" binding:"required"`
|
||||
NewTipo string `json:"new_tipo" binding:"required"`
|
||||
}
|
||||
if err := c.ShouldBindJSON(&req); err != nil {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": "Invalid request", "message": err.Error()})
|
||||
|
|
@ -684,96 +629,41 @@ func PatchEntityTipo(c *gin.Context) {
|
|||
})
|
||||
}
|
||||
|
||||
// pgConnInfo holds PostgreSQL connection parameters for pg_dump
|
||||
type pgConnInfo struct {
|
||||
host, port, name, user, pass string
|
||||
}
|
||||
|
||||
// resolvePgConnInfo prefers DATABASE_URL (the variable the API container gets)
|
||||
// and falls back to the individual DB_* variables used by the workers.
|
||||
func resolvePgConnInfo() pgConnInfo {
|
||||
if dbURL := os.Getenv("DATABASE_URL"); dbURL != "" {
|
||||
if u, err := url.Parse(dbURL); err == nil && u.Hostname() != "" {
|
||||
info := pgConnInfo{
|
||||
host: u.Hostname(),
|
||||
user: u.User.Username(),
|
||||
name: strings.TrimPrefix(u.Path, "/"),
|
||||
port: u.Port(),
|
||||
}
|
||||
if p, ok := u.User.Password(); ok {
|
||||
info.pass = p
|
||||
}
|
||||
if info.port == "" {
|
||||
info.port = "5432"
|
||||
}
|
||||
if info.user == "" {
|
||||
info.user = "postgres"
|
||||
}
|
||||
if info.name == "" {
|
||||
info.name = "postgres"
|
||||
}
|
||||
return info
|
||||
}
|
||||
}
|
||||
|
||||
info := pgConnInfo{
|
||||
host: os.Getenv("DB_HOST"),
|
||||
port: os.Getenv("DB_PORT"),
|
||||
name: os.Getenv("DB_NAME"),
|
||||
user: os.Getenv("DB_USER"),
|
||||
pass: os.Getenv("DB_PASS"),
|
||||
}
|
||||
if info.host == "" {
|
||||
info.host = "db"
|
||||
}
|
||||
if info.port == "" {
|
||||
info.port = "5432"
|
||||
}
|
||||
if info.name == "" {
|
||||
info.name = "rss"
|
||||
}
|
||||
if info.user == "" {
|
||||
info.user = "rss"
|
||||
}
|
||||
return info
|
||||
}
|
||||
|
||||
// flushWriter flushes the underlying response writer after every write so that
|
||||
// large backups stream to the client instead of being buffered in RAM.
|
||||
type flushWriter struct {
|
||||
w io.Writer
|
||||
}
|
||||
|
||||
func (fw flushWriter) Write(p []byte) (int, error) {
|
||||
n, err := fw.w.Write(p)
|
||||
if f, ok := fw.w.(http.Flusher); ok {
|
||||
f.Flush()
|
||||
}
|
||||
return n, err
|
||||
}
|
||||
|
||||
// BackupDatabase runs pg_dump and returns the SQL as a downloadable file.
|
||||
// The output is streamed to the client so arbitrarily large dumps do not
|
||||
// exhaust the container memory.
|
||||
// BackupDatabase runs pg_dump and returns the SQL as a downloadable file
|
||||
func BackupDatabase(c *gin.Context) {
|
||||
info := resolvePgConnInfo()
|
||||
dbHost := os.Getenv("DB_HOST")
|
||||
if dbHost == "" {
|
||||
dbHost = "db"
|
||||
}
|
||||
dbPort := os.Getenv("DB_PORT")
|
||||
if dbPort == "" {
|
||||
dbPort = "5432"
|
||||
}
|
||||
dbName := os.Getenv("DB_NAME")
|
||||
if dbName == "" {
|
||||
dbName = "rss"
|
||||
}
|
||||
dbUser := os.Getenv("DB_USER")
|
||||
if dbUser == "" {
|
||||
dbUser = "rss"
|
||||
}
|
||||
dbPass := os.Getenv("DB_PASS")
|
||||
|
||||
cmd := exec.Command("pg_dump",
|
||||
"-h", info.host,
|
||||
"-p", info.port,
|
||||
"-U", info.user,
|
||||
"-d", info.name,
|
||||
"-h", dbHost,
|
||||
"-p", dbPort,
|
||||
"-U", dbUser,
|
||||
"-d", dbName,
|
||||
"--no-password",
|
||||
"--format=plain",
|
||||
)
|
||||
cmd.Env = append(os.Environ(), fmt.Sprintf("PGPASSWORD=%s", info.pass))
|
||||
cmd.Env = append(os.Environ(), fmt.Sprintf("PGPASSWORD=%s", dbPass))
|
||||
|
||||
pr, pw := io.Pipe()
|
||||
cmd.Stdout = pw
|
||||
var out bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
cmd.Stdout = &out
|
||||
cmd.Stderr = &stderr
|
||||
|
||||
if err := cmd.Start(); err != nil {
|
||||
if err := cmd.Run(); err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{
|
||||
"error": "pg_dump failed",
|
||||
"details": stderr.String(),
|
||||
|
|
@ -785,40 +675,37 @@ func BackupDatabase(c *gin.Context) {
|
|||
c.Header("Content-Type", "application/octet-stream")
|
||||
c.Header("Content-Disposition", fmt.Sprintf("attachment; filename=%s", filename))
|
||||
c.Header("Cache-Control", "no-cache")
|
||||
c.Status(http.StatusOK)
|
||||
|
||||
fw := flushWriter{w: c.Writer}
|
||||
copyDone := make(chan struct{})
|
||||
go func() {
|
||||
defer close(copyDone)
|
||||
_, _ = io.Copy(fw, pr)
|
||||
}()
|
||||
|
||||
err := cmd.Wait()
|
||||
pw.Close()
|
||||
<-copyDone
|
||||
|
||||
if err != nil {
|
||||
log.Printf("BackupDatabase: pg_dump failed: %v: %s", err, stderr.String())
|
||||
c.Writer.Flush()
|
||||
return
|
||||
}
|
||||
c.Writer.Flush()
|
||||
c.Data(http.StatusOK, "application/octet-stream", out.Bytes())
|
||||
}
|
||||
|
||||
// BackupNewsZipped performs a pg_dump of news tables and returns a ZIP file.
|
||||
// pg_dump output is streamed into the ZIP as it is produced, keeping memory usage flat.
|
||||
// BackupNewsZipped performs a pg_dump of news tables and returns a ZIP file
|
||||
func BackupNewsZipped(c *gin.Context) {
|
||||
info := resolvePgConnInfo()
|
||||
dbHost := os.Getenv("DB_HOST")
|
||||
if dbHost == "" {
|
||||
dbHost = "db"
|
||||
}
|
||||
dbPort := os.Getenv("DB_PORT")
|
||||
if dbPort == "" {
|
||||
dbPort = "5432"
|
||||
}
|
||||
dbName := os.Getenv("DB_NAME")
|
||||
if dbName == "" {
|
||||
dbName = "rss"
|
||||
}
|
||||
dbUser := os.Getenv("DB_USER")
|
||||
if dbUser == "" {
|
||||
dbUser = "rss"
|
||||
}
|
||||
dbPass := os.Getenv("DB_PASS")
|
||||
|
||||
// Tables to backup
|
||||
tables := []string{"noticias", "traducciones", "tags", "tags_noticia"}
|
||||
|
||||
args := []string{
|
||||
"-h", info.host,
|
||||
"-p", info.port,
|
||||
"-U", info.user,
|
||||
"-d", info.name,
|
||||
"-h", dbHost,
|
||||
"-p", dbPort,
|
||||
"-U", dbUser,
|
||||
"-d", dbName,
|
||||
"--no-password",
|
||||
}
|
||||
|
||||
|
|
@ -827,14 +714,14 @@ func BackupNewsZipped(c *gin.Context) {
|
|||
}
|
||||
|
||||
cmd := exec.Command("pg_dump", args...)
|
||||
cmd.Env = append(os.Environ(), fmt.Sprintf("PGPASSWORD=%s", info.pass))
|
||||
cmd.Env = append(os.Environ(), fmt.Sprintf("PGPASSWORD=%s", dbPass))
|
||||
|
||||
pr, pw := io.Pipe()
|
||||
cmd.Stdout = pw
|
||||
var sqlOut bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
cmd.Stdout = &sqlOut
|
||||
cmd.Stderr = &stderr
|
||||
|
||||
if err := cmd.Start(); err != nil {
|
||||
if err := cmd.Run(); err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{
|
||||
"error": "pg_dump failed",
|
||||
"details": stderr.String(),
|
||||
|
|
@ -842,142 +729,32 @@ func BackupNewsZipped(c *gin.Context) {
|
|||
return
|
||||
}
|
||||
|
||||
filename := fmt.Sprintf("backup_noticias_%s.zip", time.Now().Format("2006-01-02_15-04-05"))
|
||||
c.Header("Content-Type", "application/zip")
|
||||
c.Header("Content-Disposition", fmt.Sprintf("attachment; filename=%s", filename))
|
||||
c.Header("Cache-Control", "no-cache")
|
||||
c.Status(http.StatusOK)
|
||||
|
||||
fw := flushWriter{w: c.Writer}
|
||||
zw := zip.NewWriter(fw)
|
||||
// Create ZIP
|
||||
buf := new(bytes.Buffer)
|
||||
zw := zip.NewWriter(buf)
|
||||
|
||||
sqlFileName := fmt.Sprintf("backup_noticias_%s.sql", time.Now().Format("2006-01-02"))
|
||||
f, err := zw.Create(sqlFileName)
|
||||
if err != nil {
|
||||
log.Printf("BackupNewsZipped: failed to create ZIP entry: %v", err)
|
||||
pw.Close()
|
||||
pr.Close()
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to create ZIP entry", "message": err.Error()})
|
||||
return
|
||||
}
|
||||
|
||||
copyDone := make(chan struct{})
|
||||
go func() {
|
||||
defer close(copyDone)
|
||||
_, _ = io.Copy(f, pr)
|
||||
}()
|
||||
|
||||
waitErr := cmd.Wait()
|
||||
pw.Close()
|
||||
<-copyDone
|
||||
closeErr := zw.Close()
|
||||
c.Writer.Flush()
|
||||
|
||||
if waitErr != nil || closeErr != nil {
|
||||
log.Printf("BackupNewsZipped: pg_dump=%v zip=%v: %s", waitErr, closeErr, stderr.String())
|
||||
_, err = f.Write(sqlOut.Bytes())
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to write to ZIP", "message": err.Error()})
|
||||
return
|
||||
}
|
||||
|
||||
if err := zw.Close(); err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to close ZIP writer", "message": err.Error()})
|
||||
return
|
||||
}
|
||||
|
||||
filename := fmt.Sprintf("backup_noticias_%s.zip", time.Now().Format("2006-01-02_15-04-05"))
|
||||
c.Header("Content-Type", "application/zip")
|
||||
c.Header("Content-Disposition", fmt.Sprintf("attachment; filename=%s", filename))
|
||||
c.Header("Cache-Control", "no-cache")
|
||||
c.Data(http.StatusOK, "application/zip", buf.Bytes())
|
||||
}
|
||||
|
||||
// RestoreDatabase accepts an uploaded .sql or .zip backup and restores it via psql
|
||||
func RestoreDatabase(c *gin.Context) {
|
||||
file, err := c.FormFile("file")
|
||||
if err != nil {
|
||||
c.JSON(http.StatusBadRequest, gin.H{
|
||||
"error": "No file uploaded",
|
||||
"message": "Se requiere un archivo .sql o .zip",
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
uploaded, err := file.Open()
|
||||
if err != nil {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": "Failed to open uploaded file"})
|
||||
return
|
||||
}
|
||||
defer uploaded.Close()
|
||||
|
||||
data, err := io.ReadAll(uploaded)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": "Failed to read uploaded file"})
|
||||
return
|
||||
}
|
||||
|
||||
var sqlData []byte
|
||||
lowerName := strings.ToLower(file.Filename)
|
||||
switch {
|
||||
case strings.HasSuffix(lowerName, ".sql"):
|
||||
sqlData = data
|
||||
case strings.HasSuffix(lowerName, ".zip"):
|
||||
zr, err := zip.NewReader(bytes.NewReader(data), int64(len(data)))
|
||||
if err != nil {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": "Invalid ZIP file", "message": err.Error()})
|
||||
return
|
||||
}
|
||||
found := false
|
||||
for _, zf := range zr.File {
|
||||
if zf.FileInfo().IsDir() || !strings.HasSuffix(strings.ToLower(zf.Name), ".sql") {
|
||||
continue
|
||||
}
|
||||
rc, err := zf.Open()
|
||||
if err != nil {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": "Failed to open SQL inside ZIP", "message": err.Error()})
|
||||
return
|
||||
}
|
||||
sqlData, err = io.ReadAll(rc)
|
||||
rc.Close()
|
||||
if err != nil {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": "Failed to read SQL inside ZIP", "message": err.Error()})
|
||||
return
|
||||
}
|
||||
found = true
|
||||
break
|
||||
}
|
||||
if !found {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": "No SQL file found inside ZIP"})
|
||||
return
|
||||
}
|
||||
default:
|
||||
c.JSON(http.StatusBadRequest, gin.H{
|
||||
"error": "Unsupported file type",
|
||||
"message": "Solo se permiten archivos .sql o .zip",
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
if len(sqlData) == 0 {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": "Empty backup file"})
|
||||
return
|
||||
}
|
||||
|
||||
info := resolvePgConnInfo()
|
||||
|
||||
cmd := exec.Command("psql",
|
||||
"-h", info.host,
|
||||
"-p", info.port,
|
||||
"-U", info.user,
|
||||
"-d", info.name,
|
||||
"--no-password",
|
||||
"-v", "ON_ERROR_STOP=1",
|
||||
)
|
||||
cmd.Env = append(os.Environ(), fmt.Sprintf("PGPASSWORD=%s", info.pass))
|
||||
cmd.Stdin = bytes.NewReader(sqlData)
|
||||
|
||||
var out bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
cmd.Stdout = &out
|
||||
cmd.Stderr = &stderr
|
||||
|
||||
if err := cmd.Run(); err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{
|
||||
"error": "Restore failed",
|
||||
"details": stderr.String(),
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
c.JSON(http.StatusOK, gin.H{
|
||||
"message": "Base de datos restaurada correctamente",
|
||||
"filename": file.Filename,
|
||||
"output": out.String(),
|
||||
})
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,245 +0,0 @@
|
|||
package handlers
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/rss2/backend/internal/db"
|
||||
"github.com/rss2/backend/internal/models"
|
||||
)
|
||||
|
||||
type Alerta struct {
|
||||
ID int64 `json:"id"`
|
||||
Valor string `json:"valor"`
|
||||
Tipo string `json:"tipo"`
|
||||
Periodo string `json:"periodo"`
|
||||
Hits int `json:"hits"`
|
||||
Baseline float64 `json:"baseline"`
|
||||
Ratio float64 `json:"ratio"`
|
||||
Status string `json:"status"`
|
||||
CreatedAt string `json:"created_at"`
|
||||
}
|
||||
|
||||
func GetAlertas(c *gin.Context) {
|
||||
status := c.Query("status")
|
||||
limitStr := c.DefaultQuery("limit", "100")
|
||||
|
||||
limit, _ := strconv.Atoi(limitStr)
|
||||
if limit < 1 || limit > 500 {
|
||||
limit = 100
|
||||
}
|
||||
|
||||
query := `
|
||||
SELECT id, valor, tipo, periodo::text, hits, baseline, ratio, status, to_char(created_at, 'YYYY-MM-DD HH24:MI') AS created_at
|
||||
FROM alertas
|
||||
`
|
||||
args := []interface{}{}
|
||||
if status != "" {
|
||||
query += fmt.Sprintf(" WHERE status = $%d", len(args)+1)
|
||||
args = append(args, status)
|
||||
}
|
||||
query += fmt.Sprintf(" ORDER BY periodo DESC, ratio DESC, id DESC LIMIT $%d", len(args)+1)
|
||||
args = append(args, limit)
|
||||
|
||||
rows, err := db.GetPool().Query(c.Request.Context(), query, args...)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, models.ErrorResponse{Error: "Failed to get alerts", Message: err.Error()})
|
||||
return
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
alertas := []Alerta{}
|
||||
for rows.Next() {
|
||||
var a Alerta
|
||||
if err := rows.Scan(&a.ID, &a.Valor, &a.Tipo, &a.Periodo, &a.Hits, &a.Baseline, &a.Ratio, &a.Status, &a.CreatedAt); err != nil {
|
||||
continue
|
||||
}
|
||||
alertas = append(alertas, a)
|
||||
}
|
||||
|
||||
var nuevas int
|
||||
db.GetPool().QueryRow(c.Request.Context(), "SELECT COUNT(*)::int FROM alertas WHERE status = 'nueva'").Scan(&nuevas)
|
||||
|
||||
c.JSON(http.StatusOK, gin.H{
|
||||
"alertas": alertas,
|
||||
"total": len(alertas),
|
||||
"nuevas": nuevas,
|
||||
})
|
||||
}
|
||||
|
||||
func MarkAlertaRead(c *gin.Context) {
|
||||
idStr := c.Param("id")
|
||||
id, err := strconv.ParseInt(idStr, 10, 64)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusBadRequest, models.ErrorResponse{Error: "Invalid alert id"})
|
||||
return
|
||||
}
|
||||
|
||||
_, err = db.GetPool().Exec(c.Request.Context(), "UPDATE alertas SET status = 'leida' WHERE id = $1", id)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, models.ErrorResponse{Error: "Failed to update alert", Message: err.Error()})
|
||||
return
|
||||
}
|
||||
|
||||
c.JSON(http.StatusOK, gin.H{"ok": true})
|
||||
}
|
||||
|
||||
func MarkAllAlertasRead(c *gin.Context) {
|
||||
_, err := db.GetPool().Exec(c.Request.Context(), "UPDATE alertas SET status = 'leida' WHERE status = 'nueva'")
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, models.ErrorResponse{Error: "Failed to update alerts", Message: err.Error()})
|
||||
return
|
||||
}
|
||||
c.JSON(http.StatusOK, gin.H{"ok": true})
|
||||
}
|
||||
|
||||
func ScanAlertasAdmin(c *gin.Context) {
|
||||
inserted, err := RunAlertScan(c.Request.Context())
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, models.ErrorResponse{Error: "Scan failed", Message: err.Error()})
|
||||
return
|
||||
}
|
||||
c.JSON(http.StatusOK, gin.H{"ok": true, "new_alerts": inserted})
|
||||
}
|
||||
|
||||
// RunAlertScan busca conceptos (persona, lugar, organizacion, tema) cuya
|
||||
// actividad supera significativamente su media de los días anteriores.
|
||||
//
|
||||
// En lugar de comparar siempre contra CURRENT_DATE (que se queda vacío si la
|
||||
// cadena de traducción/NER va por detrás de la ingesta), se toma como día de
|
||||
// referencia el día más reciente que tenga datos de tags etiquetados y se
|
||||
// compara contra la media de los ALERTS_LOOKBACK_DAYS días previos, contando
|
||||
// los días sin actividad como 0.
|
||||
func RunAlertScan(ctx context.Context) (int, error) {
|
||||
minHits := 5
|
||||
minRatio := 5.0
|
||||
minBaseline := 2.0
|
||||
lookbackDays := 8
|
||||
if v := os.Getenv("ALERTS_MIN_HITS"); v != "" {
|
||||
if n, err := strconv.Atoi(v); err == nil && n > 0 {
|
||||
minHits = n
|
||||
}
|
||||
}
|
||||
if v := os.Getenv("ALERTS_MIN_RATIO"); v != "" {
|
||||
if f, err := strconv.ParseFloat(v, 64); err == nil && f > 0 {
|
||||
minRatio = f
|
||||
}
|
||||
}
|
||||
if v := os.Getenv("ALERTS_MIN_BASELINE"); v != "" {
|
||||
if f, err := strconv.ParseFloat(v, 64); err == nil && f > 0 {
|
||||
minBaseline = f
|
||||
}
|
||||
}
|
||||
if v := os.Getenv("ALERTS_LOOKBACK_DAYS"); v != "" {
|
||||
if n, err := strconv.Atoi(v); err == nil && n > 0 {
|
||||
lookbackDays = n
|
||||
}
|
||||
}
|
||||
|
||||
query := `
|
||||
WITH daily AS (
|
||||
SELECT t.id AS tag_id, t.valor, t.tipo, n.fecha::date AS dia,
|
||||
COUNT(DISTINCT n.id) AS cnt
|
||||
FROM tags_noticia tn
|
||||
JOIN tags t ON tn.tag_id = t.id
|
||||
JOIN traducciones tr ON tn.traduccion_id = tr.id
|
||||
JOIN noticias n ON tr.noticia_id = n.id
|
||||
WHERE n.fecha >= CURRENT_DATE - 30
|
||||
AND n.fecha <= CURRENT_DATE
|
||||
AND t.tipo IN ('persona', 'lugar', 'organizacion', 'tema')
|
||||
GROUP BY t.id, t.valor, t.tipo, n.fecha::date
|
||||
),
|
||||
ref AS (
|
||||
SELECT MAX(dia) AS ref_date FROM daily
|
||||
),
|
||||
prev_series AS (
|
||||
SELECT generate_series((SELECT ref_date - $3::int FROM ref),
|
||||
(SELECT ref_date - 1 FROM ref), '1 day')::date AS dia
|
||||
),
|
||||
today AS (
|
||||
SELECT tag_id, valor, tipo, SUM(cnt)::int AS hits
|
||||
FROM daily
|
||||
WHERE dia = (SELECT ref_date FROM ref)
|
||||
GROUP BY tag_id, valor, tipo
|
||||
),
|
||||
base AS (
|
||||
SELECT t.tag_id, t.valor, t.tipo,
|
||||
AVG(COALESCE(d.cnt, 0))::float AS baseline
|
||||
FROM today t
|
||||
CROSS JOIN prev_series s
|
||||
LEFT JOIN daily d ON d.tag_id = t.tag_id AND d.dia = s.dia
|
||||
GROUP BY t.tag_id, t.valor, t.tipo
|
||||
)
|
||||
SELECT t.valor, t.tipo, t.hits, b.baseline,
|
||||
(t.hits::float / b.baseline) AS ratio,
|
||||
(SELECT ref_date FROM ref)::date AS periodo
|
||||
FROM today t
|
||||
JOIN base b USING (tag_id)
|
||||
WHERE t.hits >= $1
|
||||
AND b.baseline >= $4
|
||||
AND (t.hits::float / b.baseline) >= $2
|
||||
`
|
||||
|
||||
rows, err := db.GetPool().Query(ctx, query, minHits, minRatio, lookbackDays, minBaseline)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
inserted := 0
|
||||
for rows.Next() {
|
||||
var valor, tipo string
|
||||
var hits int
|
||||
var baseline, ratio float64
|
||||
var periodo time.Time
|
||||
if err := rows.Scan(&valor, &tipo, &hits, &baseline, &ratio, &periodo); err != nil {
|
||||
continue
|
||||
}
|
||||
res, err := db.GetPool().Exec(ctx, `
|
||||
INSERT INTO alertas (valor, tipo, periodo, hits, baseline, ratio)
|
||||
VALUES ($1, $2, $3, $4, $5, $6)
|
||||
ON CONFLICT (valor, tipo, periodo) DO NOTHING
|
||||
`, valor, tipo, periodo, hits, baseline, ratio)
|
||||
if err != nil {
|
||||
log.Printf("alerts: insert failed for %s (%s): %v", valor, tipo, err)
|
||||
continue
|
||||
}
|
||||
if n := res.RowsAffected(); n > 0 {
|
||||
inserted++
|
||||
}
|
||||
}
|
||||
|
||||
return inserted, nil
|
||||
}
|
||||
|
||||
// StartAlertScanner lanza una goroutine que revisa picos de actividad
|
||||
// periódicamente (cada ALERTS_SCAN_INTERVAL_MIN, por defecto 120 min).
|
||||
func StartAlertScanner() {
|
||||
interval := 120
|
||||
if v := os.Getenv("ALERTS_SCAN_INTERVAL_MIN"); v != "" {
|
||||
if n, err := strconv.Atoi(v); err == nil && n > 0 {
|
||||
interval = n
|
||||
}
|
||||
}
|
||||
|
||||
go func() {
|
||||
time.Sleep(45 * time.Second)
|
||||
for {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute)
|
||||
inserted, err := RunAlertScan(ctx)
|
||||
cancel()
|
||||
if err != nil {
|
||||
log.Printf("alerts: scan error: %v", err)
|
||||
} else {
|
||||
log.Printf("alerts: scan finished, %d new alerts", inserted)
|
||||
}
|
||||
time.Sleep(time.Duration(interval) * time.Minute)
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
|
@ -2,22 +2,18 @@ package handlers
|
|||
|
||||
import (
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/rss2/backend/internal/auth"
|
||||
"github.com/golang-jwt/jwt/v5"
|
||||
"github.com/rss2/backend/internal/config"
|
||||
"github.com/rss2/backend/internal/db"
|
||||
"github.com/rss2/backend/internal/models"
|
||||
"golang.org/x/crypto/bcrypt"
|
||||
)
|
||||
|
||||
// CheckFirstUser checks if this is the first user registration
|
||||
// @Summary Check if first user
|
||||
// @Description Returns whether this is the first user being registered (no users exist yet)
|
||||
// @Tags auth
|
||||
// @Produce json
|
||||
// @Success 200 {object} map[string]interface{} "is_first_user and total_users"
|
||||
// @Failure 500 {object} models.ErrorResponse
|
||||
// @Router /auth/check-first-user [get]
|
||||
var jwtSecret []byte
|
||||
|
||||
func CheckFirstUser(c *gin.Context) {
|
||||
var count int
|
||||
err := db.GetPool().QueryRow(c.Request.Context(), "SELECT COUNT(*) FROM users").Scan(&count)
|
||||
|
|
@ -28,18 +24,18 @@ func CheckFirstUser(c *gin.Context) {
|
|||
c.JSON(http.StatusOK, gin.H{"is_first_user": count == 0, "total_users": count})
|
||||
}
|
||||
|
||||
// Login authenticates a user and returns a JWT token
|
||||
// @Summary User login
|
||||
// @Description Authenticate with email and password to receive a JWT token
|
||||
// @Tags auth
|
||||
// @Accept json
|
||||
// @Produce json
|
||||
// @Param request body models.LoginRequest true "Login credentials"
|
||||
// @Success 200 {object} models.AuthResponse
|
||||
// @Failure 400 {object} models.ErrorResponse
|
||||
// @Failure 401 {object} models.ErrorResponse
|
||||
// @Failure 500 {object} models.ErrorResponse
|
||||
// @Router /auth/login [post]
|
||||
func InitAuth(secret string) {
|
||||
jwtSecret = []byte(secret)
|
||||
}
|
||||
|
||||
type Claims struct {
|
||||
UserID int64 `json:"user_id"`
|
||||
Email string `json:"email"`
|
||||
Username string `json:"username"`
|
||||
IsAdmin bool `json:"is_admin"`
|
||||
jwt.RegisteredClaims
|
||||
}
|
||||
|
||||
func Login(c *gin.Context) {
|
||||
var req models.LoginRequest
|
||||
if err := c.ShouldBindJSON(&req); err != nil {
|
||||
|
|
@ -64,7 +60,20 @@ func Login(c *gin.Context) {
|
|||
return
|
||||
}
|
||||
|
||||
tokenString, err := auth.GenerateToken(user.ID, user.Email, user.Username, user.IsAdmin)
|
||||
expirationTime := time.Now().Add(24 * time.Hour)
|
||||
claims := &Claims{
|
||||
UserID: user.ID,
|
||||
Email: user.Email,
|
||||
Username: user.Username,
|
||||
IsAdmin: user.IsAdmin,
|
||||
RegisteredClaims: jwt.RegisteredClaims{
|
||||
ExpiresAt: jwt.NewNumericDate(expirationTime),
|
||||
IssuedAt: jwt.NewNumericDate(time.Now()),
|
||||
},
|
||||
}
|
||||
|
||||
token := jwt.NewWithClaims(jwt.SigningMethodHS256, claims)
|
||||
tokenString, err := token.SignedString(jwtSecret)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, models.ErrorResponse{Error: "Failed to generate token"})
|
||||
return
|
||||
|
|
@ -76,17 +85,6 @@ func Login(c *gin.Context) {
|
|||
})
|
||||
}
|
||||
|
||||
// Register creates a new user account
|
||||
// @Summary Register new user
|
||||
// @Description Create a new user account (first user becomes admin automatically)
|
||||
// @Tags auth
|
||||
// @Accept json
|
||||
// @Produce json
|
||||
// @Param request body models.RegisterRequest true "Registration details"
|
||||
// @Success 201 {object} models.AuthResponse
|
||||
// @Failure 400 {object} models.ErrorResponse
|
||||
// @Failure 500 {object} models.ErrorResponse
|
||||
// @Router /auth/register [post]
|
||||
func Register(c *gin.Context) {
|
||||
var req models.RegisterRequest
|
||||
if err := c.ShouldBindJSON(&req); err != nil {
|
||||
|
|
@ -129,7 +127,20 @@ func Register(c *gin.Context) {
|
|||
return
|
||||
}
|
||||
|
||||
tokenString, err := auth.GenerateToken(user.ID, user.Email, user.Username, user.IsAdmin)
|
||||
expirationTime := time.Now().Add(24 * time.Hour)
|
||||
claims := &Claims{
|
||||
UserID: user.ID,
|
||||
Email: user.Email,
|
||||
Username: user.Username,
|
||||
IsAdmin: user.IsAdmin,
|
||||
RegisteredClaims: jwt.RegisteredClaims{
|
||||
ExpiresAt: jwt.NewNumericDate(expirationTime),
|
||||
IssuedAt: jwt.NewNumericDate(time.Now()),
|
||||
},
|
||||
}
|
||||
|
||||
token := jwt.NewWithClaims(jwt.SigningMethodHS256, claims)
|
||||
tokenString, err := token.SignedString(jwtSecret)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, models.ErrorResponse{Error: "Failed to generate token"})
|
||||
return
|
||||
|
|
@ -142,16 +153,6 @@ func Register(c *gin.Context) {
|
|||
})
|
||||
}
|
||||
|
||||
// GetCurrentUser returns the current authenticated user
|
||||
// @Summary Get current user
|
||||
// @Description Returns the currently authenticated user's profile
|
||||
// @Tags auth
|
||||
// @Produce json
|
||||
// @Security BearerAuth
|
||||
// @Success 200 {object} models.User
|
||||
// @Failure 401 {object} models.ErrorResponse
|
||||
// @Failure 404 {object} models.ErrorResponse
|
||||
// @Router /auth/me [get]
|
||||
func GetCurrentUser(c *gin.Context) {
|
||||
userVal, exists := c.Get("user")
|
||||
if !exists {
|
||||
|
|
@ -159,7 +160,7 @@ func GetCurrentUser(c *gin.Context) {
|
|||
return
|
||||
}
|
||||
|
||||
claims := userVal.(*auth.Claims)
|
||||
claims := userVal.(*Claims)
|
||||
|
||||
var user models.User
|
||||
err := db.GetPool().QueryRow(c.Request.Context(), `
|
||||
|
|
@ -175,3 +176,8 @@ func GetCurrentUser(c *gin.Context) {
|
|||
|
||||
c.JSON(http.StatusOK, user)
|
||||
}
|
||||
|
||||
func init() {
|
||||
cfg := config.Load()
|
||||
InitAuth(cfg.SecretKey)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,74 +0,0 @@
|
|||
package handlers
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/rss2/backend/internal/auth"
|
||||
"github.com/rss2/backend/internal/models"
|
||||
)
|
||||
|
||||
func init() {
|
||||
gin.SetMode(gin.TestMode)
|
||||
auth.SetJWTSecret("test-secret-key-12345")
|
||||
}
|
||||
|
||||
// TestLoginInvalidRequest tests that an empty request body returns 400
|
||||
func TestLoginInvalidRequest(t *testing.T) {
|
||||
router := gin.New()
|
||||
router.POST("/auth/login", Login)
|
||||
|
||||
body := []byte(`{}`)
|
||||
req, _ := http.NewRequest("POST", "/auth/login", bytes.NewBuffer(body))
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
|
||||
w := httptest.NewRecorder()
|
||||
router.ServeHTTP(w, req)
|
||||
|
||||
if w.Code != http.StatusBadRequest {
|
||||
t.Errorf("expected status 400, got %d", w.Code)
|
||||
}
|
||||
|
||||
var resp models.ErrorResponse
|
||||
json.Unmarshal(w.Body.Bytes(), &resp)
|
||||
|
||||
if resp.Error != "Invalid request" {
|
||||
t.Errorf("expected 'Invalid request', got %s", resp.Error)
|
||||
}
|
||||
}
|
||||
|
||||
// TestRegisterInvalidRequest tests that an empty request body returns 400
|
||||
func TestRegisterInvalidRequest(t *testing.T) {
|
||||
router := gin.New()
|
||||
router.POST("/auth/register", Register)
|
||||
|
||||
body := []byte(`{}`)
|
||||
req, _ := http.NewRequest("POST", "/auth/register", bytes.NewBuffer(body))
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
|
||||
w := httptest.NewRecorder()
|
||||
router.ServeHTTP(w, req)
|
||||
|
||||
if w.Code != http.StatusBadRequest {
|
||||
t.Errorf("expected status 400, got %d", w.Code)
|
||||
}
|
||||
}
|
||||
|
||||
// TestGetCurrentUserUnauthorized tests that unauthenticated request returns 401
|
||||
func TestGetCurrentUserUnauthorized(t *testing.T) {
|
||||
router := gin.New()
|
||||
router.GET("/auth/me", GetCurrentUser)
|
||||
|
||||
req, _ := http.NewRequest("GET", "/auth/me", nil)
|
||||
|
||||
w := httptest.NewRecorder()
|
||||
router.ServeHTTP(w, req)
|
||||
|
||||
if w.Code != http.StatusUnauthorized {
|
||||
t.Errorf("expected status 401, got %d", w.Code)
|
||||
}
|
||||
}
|
||||
|
|
@ -3,54 +3,46 @@ package handlers
|
|||
import (
|
||||
"context"
|
||||
"encoding/csv"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/rss2/backend/internal/cache"
|
||||
"github.com/rss2/backend/internal/db"
|
||||
"github.com/rss2/backend/internal/models"
|
||||
)
|
||||
|
||||
// GetFeeds returns paginated feeds with optional filters
|
||||
// @Summary List feeds
|
||||
// @Description Returns paginated feeds with optional filtering by active status, category, and country
|
||||
// @Tags feeds
|
||||
// @Produce json
|
||||
// @Param page query int false "Page number" default(1) minimum(1)
|
||||
// @Param per_page query int false "Items per page" default(50) minimum(1) maximum(100)
|
||||
// @Param activo query string false "Active status filter"
|
||||
// @Param categoria_id query string false "Category ID filter"
|
||||
// @Param pais_id query string false "Country ID filter"
|
||||
// @Success 200 {object} map[string]interface{} "feeds, total, page, per_page, total_pages"
|
||||
// @Failure 500 {object} models.ErrorResponse
|
||||
// @Router /feeds [get]
|
||||
type FeedResponse struct {
|
||||
ID int64 `json:"id"`
|
||||
Nombre string `json:"nombre"`
|
||||
Descripcion *string `json:"descripcion"`
|
||||
URL string `json:"url"`
|
||||
CategoriaID *int64 `json:"categoria_id"`
|
||||
PaisID *int64 `json:"pais_id"`
|
||||
Idioma *string `json:"idioma"`
|
||||
Activo bool `json:"activo"`
|
||||
Fallos *int64 `json:"fallos"`
|
||||
LastError *string `json:"last_error"`
|
||||
FuenteURLID *int64 `json:"fuente_url_id"`
|
||||
}
|
||||
|
||||
func GetFeeds(c *gin.Context) {
|
||||
page, _ := strconv.Atoi(c.DefaultQuery("page", "1"))
|
||||
perPage, _ := strconv.Atoi(c.DefaultQuery("per_page", "50"))
|
||||
page, perPage = validatePageParams(page, perPage, 50, 100)
|
||||
activo := c.Query("activo")
|
||||
categoriaID := c.Query("categoria_id")
|
||||
paisID := c.Query("pais_id")
|
||||
|
||||
offset := (page - 1) * perPage
|
||||
|
||||
// Cache key based on filters
|
||||
filterStr := fmt.Sprintf("activo_%s_cat_%s_pais_%s", activo, categoriaID, paisID)
|
||||
cacheKey := cache.FeedListKey(filterStr, page, perPage)
|
||||
|
||||
ctx := c.Request.Context()
|
||||
|
||||
// Try to get from cache
|
||||
if cached, err := cache.Get(ctx, cacheKey); err == nil && cached != "" {
|
||||
c.Data(http.StatusOK, "application/json", []byte(cached))
|
||||
return
|
||||
if page < 1 {
|
||||
page = 1
|
||||
}
|
||||
if perPage < 1 || perPage > 100 {
|
||||
perPage = 50
|
||||
}
|
||||
|
||||
offset := (page - 1) * perPage
|
||||
|
||||
where := "1=1"
|
||||
args := []interface{}{}
|
||||
|
|
@ -74,7 +66,7 @@ func GetFeeds(c *gin.Context) {
|
|||
|
||||
var total int
|
||||
countQuery := fmt.Sprintf("SELECT COUNT(*) FROM feeds WHERE %s", where)
|
||||
err := db.GetPool().QueryRow(ctx, countQuery, args...).Scan(&total)
|
||||
err := db.GetPool().QueryRow(c.Request.Context(), countQuery, args...).Scan(&total)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, models.ErrorResponse{Error: "Failed to count feeds", Message: err.Error()})
|
||||
return
|
||||
|
|
@ -82,7 +74,7 @@ func GetFeeds(c *gin.Context) {
|
|||
|
||||
sqlQuery := fmt.Sprintf(`
|
||||
SELECT f.id, f.nombre, f.descripcion, f.url,
|
||||
f.categoria_id, f.pais_id, f.idioma, f.activo, f.fallos, f.last_error, f.last_fetched,
|
||||
f.categoria_id, f.pais_id, f.idioma, f.activo, f.fallos, f.last_error,
|
||||
c.nombre AS categoria, p.nombre AS pais,
|
||||
(SELECT COUNT(*) FROM noticias n WHERE n.fuente_nombre = f.nombre) as noticias_count
|
||||
FROM feeds f
|
||||
|
|
@ -95,7 +87,7 @@ func GetFeeds(c *gin.Context) {
|
|||
|
||||
args = append(args, perPage, offset)
|
||||
|
||||
rows, err := db.GetPool().Query(ctx, sqlQuery, args...)
|
||||
rows, err := db.GetPool().Query(c.Request.Context(), sqlQuery, args...)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, models.ErrorResponse{Error: "Failed to fetch feeds", Message: err.Error()})
|
||||
return
|
||||
|
|
@ -103,7 +95,7 @@ func GetFeeds(c *gin.Context) {
|
|||
defer rows.Close()
|
||||
|
||||
type FeedWithStats struct {
|
||||
models.Feed
|
||||
FeedResponse
|
||||
Categoria *string `json:"categoria"`
|
||||
Pais *string `json:"pais"`
|
||||
NoticiasCount int64 `json:"noticias_count"`
|
||||
|
|
@ -114,7 +106,7 @@ func GetFeeds(c *gin.Context) {
|
|||
var f FeedWithStats
|
||||
err := rows.Scan(
|
||||
&f.ID, &f.Nombre, &f.Descripcion, &f.URL,
|
||||
&f.CategoriaID, &f.PaisID, &f.Idioma, &f.Activo, &f.Fallos, &f.LastError, &f.LastFetched,
|
||||
&f.CategoriaID, &f.PaisID, &f.Idioma, &f.Activo, &f.Fallos, &f.LastError,
|
||||
&f.Categoria, &f.Pais, &f.NoticiasCount,
|
||||
)
|
||||
if err != nil {
|
||||
|
|
@ -125,20 +117,13 @@ func GetFeeds(c *gin.Context) {
|
|||
|
||||
totalPages := (total + perPage - 1) / perPage
|
||||
|
||||
response := gin.H{
|
||||
c.JSON(http.StatusOK, gin.H{
|
||||
"feeds": feeds,
|
||||
"total": total,
|
||||
"page": page,
|
||||
"per_page": perPage,
|
||||
"total_pages": totalPages,
|
||||
}
|
||||
|
||||
// Cache the response
|
||||
if data, err := json.Marshal(response); err == nil {
|
||||
cache.Set(ctx, cacheKey, string(data), cache.TTLMedium)
|
||||
}
|
||||
|
||||
c.JSON(http.StatusOK, response)
|
||||
})
|
||||
}
|
||||
|
||||
func GetFeedByID(c *gin.Context) {
|
||||
|
|
@ -148,12 +133,12 @@ func GetFeedByID(c *gin.Context) {
|
|||
return
|
||||
}
|
||||
|
||||
var f models.Feed
|
||||
var f FeedResponse
|
||||
err = db.GetPool().QueryRow(c.Request.Context(), `
|
||||
SELECT id, nombre, descripcion, url, categoria_id, pais_id, idioma, activo, fallos, last_fetched
|
||||
SELECT id, nombre, descripcion, url, categoria_id, pais_id, idioma, activo, fallos
|
||||
FROM feeds WHERE id = $1`, id).Scan(
|
||||
&f.ID, &f.Nombre, &f.Descripcion, &f.URL,
|
||||
&f.CategoriaID, &f.PaisID, &f.Idioma, &f.Activo, &f.Fallos, &f.LastFetched,
|
||||
&f.CategoriaID, &f.PaisID, &f.Idioma, &f.Activo, &f.Fallos,
|
||||
)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusNotFound, models.ErrorResponse{Error: "Feed not found"})
|
||||
|
|
@ -192,9 +177,6 @@ func CreateFeed(c *gin.Context) {
|
|||
return
|
||||
}
|
||||
|
||||
// Invalidate feed list cache
|
||||
cache.InvalidateFeedList(c.Request.Context())
|
||||
|
||||
c.JSON(http.StatusCreated, gin.H{"id": feedID, "message": "Feed created successfully"})
|
||||
}
|
||||
|
||||
|
|
@ -245,10 +227,6 @@ func UpdateFeed(c *gin.Context) {
|
|||
return
|
||||
}
|
||||
|
||||
// Invalidate feed cache
|
||||
cache.InvalidateFeed(c.Request.Context(), id)
|
||||
cache.InvalidateFeedList(c.Request.Context())
|
||||
|
||||
c.JSON(http.StatusOK, models.SuccessResponse{Message: "Feed updated successfully"})
|
||||
}
|
||||
|
||||
|
|
@ -270,10 +248,6 @@ func DeleteFeed(c *gin.Context) {
|
|||
return
|
||||
}
|
||||
|
||||
// Invalidate feed cache
|
||||
cache.InvalidateFeed(c.Request.Context(), id)
|
||||
cache.InvalidateFeedList(c.Request.Context())
|
||||
|
||||
c.JSON(http.StatusOK, models.SuccessResponse{Message: "Feed deleted successfully"})
|
||||
}
|
||||
|
||||
|
|
@ -296,10 +270,6 @@ func ToggleFeedActive(c *gin.Context) {
|
|||
return
|
||||
}
|
||||
|
||||
// Invalidate feed cache
|
||||
cache.InvalidateFeed(c.Request.Context(), id)
|
||||
cache.InvalidateFeedList(c.Request.Context())
|
||||
|
||||
c.JSON(http.StatusOK, models.SuccessResponse{Message: "Feed toggled successfully"})
|
||||
}
|
||||
|
||||
|
|
@ -322,10 +292,6 @@ func ReactivateFeed(c *gin.Context) {
|
|||
return
|
||||
}
|
||||
|
||||
// Invalidate feed cache
|
||||
cache.InvalidateFeed(c.Request.Context(), id)
|
||||
cache.InvalidateFeedList(c.Request.Context())
|
||||
|
||||
c.JSON(http.StatusOK, models.SuccessResponse{Message: "Feed reactivated successfully"})
|
||||
}
|
||||
|
||||
|
|
@ -431,46 +397,11 @@ func ImportFeeds(c *gin.Context) {
|
|||
}
|
||||
|
||||
reader := csv.NewReader(strings.NewReader(string(content)))
|
||||
records, err := reader.ReadAll()
|
||||
_, err = reader.Read()
|
||||
if err != nil {
|
||||
c.JSON(http.StatusBadRequest, models.ErrorResponse{Error: "Invalid CSV format"})
|
||||
return
|
||||
}
|
||||
if len(records) == 0 {
|
||||
c.JSON(http.StatusBadRequest, models.ErrorResponse{Error: "Empty CSV file"})
|
||||
return
|
||||
}
|
||||
|
||||
colIndex := func(header []string, names ...string) int {
|
||||
for i, h := range header {
|
||||
h = strings.ToLower(strings.TrimSpace(h))
|
||||
for _, n := range names {
|
||||
if h == n {
|
||||
return i
|
||||
}
|
||||
}
|
||||
}
|
||||
return -1
|
||||
}
|
||||
|
||||
header := records[0]
|
||||
isHeader := colIndex(header, "url", "link", "feed", "nombre", "name", "titulo") != -1
|
||||
|
||||
var dataRows [][]string
|
||||
iNombre, iDesc, iURL, iCat, iPais, iLang, iActivo, iFallos := -1, -1, -1, -1, -1, -1, -1, -1
|
||||
if isHeader {
|
||||
iNombre = colIndex(header, "nombre", "name", "titulo", "title")
|
||||
iDesc = colIndex(header, "descripcion", "description")
|
||||
iURL = colIndex(header, "url", "link", "feed")
|
||||
iCat = colIndex(header, "categoria_id", "category_id", "categoria", "category")
|
||||
iPais = colIndex(header, "pais_id", "country_id", "pais", "country")
|
||||
iLang = colIndex(header, "idioma", "language", "lang")
|
||||
iActivo = colIndex(header, "activo", "active", "enabled")
|
||||
iFallos = colIndex(header, "fallos", "failures", "fails")
|
||||
dataRows = records[1:]
|
||||
} else {
|
||||
dataRows = records
|
||||
}
|
||||
|
||||
imported := 0
|
||||
skipped := 0
|
||||
|
|
@ -482,122 +413,73 @@ func ImportFeeds(c *gin.Context) {
|
|||
c.JSON(http.StatusInternalServerError, models.ErrorResponse{Error: "Failed to start transaction", Message: err.Error()})
|
||||
return
|
||||
}
|
||||
committed := false
|
||||
defer func() {
|
||||
if !committed {
|
||||
tx.Rollback(context.Background())
|
||||
}
|
||||
}()
|
||||
defer tx.Rollback(context.Background())
|
||||
|
||||
field := func(record []string, idx int) string {
|
||||
if idx < 0 || idx >= len(record) {
|
||||
return ""
|
||||
for {
|
||||
record, err := reader.Read()
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
failed++
|
||||
continue
|
||||
}
|
||||
return strings.TrimSpace(record[idx])
|
||||
}
|
||||
|
||||
for _, record := range dataRows {
|
||||
if len(record) == 0 {
|
||||
if len(record) < 4 {
|
||||
skipped++
|
||||
continue
|
||||
}
|
||||
|
||||
var nombre, url string
|
||||
if isHeader {
|
||||
nombre = field(record, iNombre)
|
||||
url = field(record, iURL)
|
||||
} else if len(record) == 1 {
|
||||
url = field(record, 0)
|
||||
} else if len(record) == 2 {
|
||||
a := field(record, 0)
|
||||
b := field(record, 1)
|
||||
if looksLikeURL(a) {
|
||||
url, nombre = a, b
|
||||
} else {
|
||||
nombre, url = a, b
|
||||
}
|
||||
} else {
|
||||
nombre = field(record, 1)
|
||||
url = field(record, 3)
|
||||
}
|
||||
nombre := strings.TrimSpace(record[1])
|
||||
url := strings.TrimSpace(record[3])
|
||||
|
||||
if url == "" {
|
||||
if nombre == "" || url == "" {
|
||||
skipped++
|
||||
continue
|
||||
}
|
||||
if nombre == "" {
|
||||
nombre = nombreFromURL(url)
|
||||
}
|
||||
|
||||
var descripcion *string
|
||||
var categoriaID *int64
|
||||
var paisID *int64
|
||||
var idioma *string
|
||||
activo := true
|
||||
var fallos int64
|
||||
if len(record) > 2 && strings.TrimSpace(record[2]) != "" {
|
||||
descripcionStr := strings.TrimSpace(record[2])
|
||||
descripcion = &descripcionStr
|
||||
}
|
||||
|
||||
if isHeader {
|
||||
if d := field(record, iDesc); d != "" {
|
||||
descripcion = &d
|
||||
}
|
||||
if v := field(record, iCat); v != "" {
|
||||
if id, err := strconv.ParseInt(v, 10, 64); err == nil {
|
||||
categoriaID = &id
|
||||
}
|
||||
}
|
||||
if v := field(record, iPais); v != "" {
|
||||
if id, err := strconv.ParseInt(v, 10, 64); err == nil {
|
||||
paisID = &id
|
||||
}
|
||||
}
|
||||
if l := field(record, iLang); l != "" {
|
||||
if len(l) > 2 {
|
||||
l = l[:2]
|
||||
}
|
||||
idioma = &l
|
||||
}
|
||||
if a := field(record, iActivo); a != "" {
|
||||
activo = strings.ToLower(a) == "true"
|
||||
}
|
||||
if f := field(record, iFallos); f != "" {
|
||||
if v, err := strconv.ParseInt(f, 10, 64); err == nil {
|
||||
fallos = v
|
||||
}
|
||||
}
|
||||
} else {
|
||||
if d := field(record, 2); d != "" {
|
||||
descripcion = &d
|
||||
}
|
||||
if v := field(record, 4); v != "" {
|
||||
if id, err := strconv.ParseInt(v, 10, 64); err == nil {
|
||||
categoriaID = &id
|
||||
}
|
||||
}
|
||||
if v := field(record, 6); v != "" {
|
||||
if id, err := strconv.ParseInt(v, 10, 64); err == nil {
|
||||
paisID = &id
|
||||
}
|
||||
}
|
||||
if l := field(record, 8); l != "" {
|
||||
if len(l) > 2 {
|
||||
l = l[:2]
|
||||
}
|
||||
idioma = &l
|
||||
}
|
||||
if a := field(record, 9); a != "" {
|
||||
activo = strings.ToLower(a) == "true"
|
||||
}
|
||||
if f := field(record, 10); f != "" {
|
||||
if v, err := strconv.ParseInt(f, 10, 64); err == nil {
|
||||
fallos = v
|
||||
}
|
||||
var categoriaID *int64
|
||||
if len(record) > 4 && strings.TrimSpace(record[4]) != "" {
|
||||
catID, err := strconv.ParseInt(strings.TrimSpace(record[4]), 10, 64)
|
||||
if err == nil {
|
||||
categoriaID = &catID
|
||||
}
|
||||
}
|
||||
|
||||
if _, err := tx.Exec(context.Background(), "SAVEPOINT sp"); err != nil {
|
||||
failed++
|
||||
errors = append(errors, fmt.Sprintf("Error for %s: %v", url, err))
|
||||
continue
|
||||
var paisID *int64
|
||||
if len(record) > 6 && strings.TrimSpace(record[6]) != "" {
|
||||
pID, err := strconv.ParseInt(strings.TrimSpace(record[6]), 10, 64)
|
||||
if err == nil {
|
||||
paisID = &pID
|
||||
}
|
||||
}
|
||||
|
||||
var idioma *string
|
||||
if len(record) > 8 && strings.TrimSpace(record[8]) != "" {
|
||||
lang := strings.TrimSpace(record[8])
|
||||
if len(lang) > 2 {
|
||||
lang = lang[:2]
|
||||
}
|
||||
idioma = &lang
|
||||
}
|
||||
|
||||
activo := true
|
||||
if len(record) > 9 && strings.TrimSpace(record[9]) != "" {
|
||||
activo = strings.ToLower(strings.TrimSpace(record[9])) == "true"
|
||||
}
|
||||
|
||||
var fallos int64
|
||||
if len(record) > 10 && strings.TrimSpace(record[10]) != "" {
|
||||
f, err := strconv.ParseInt(strings.TrimSpace(record[10]), 10, 64)
|
||||
if err == nil {
|
||||
fallos = f
|
||||
}
|
||||
}
|
||||
|
||||
var existingID int64
|
||||
|
|
@ -608,27 +490,22 @@ func ImportFeeds(c *gin.Context) {
|
|||
WHERE id=$8`,
|
||||
nombre, descripcion, categoriaID, paisID, idioma, activo, fallos, existingID,
|
||||
)
|
||||
if err != nil {
|
||||
failed++
|
||||
errors = append(errors, fmt.Sprintf("Error updating %s: %v", url, err))
|
||||
continue
|
||||
}
|
||||
} else {
|
||||
_, err = tx.Exec(context.Background(), `
|
||||
INSERT INTO feeds (nombre, descripcion, url, categoria_id, pais_id, idioma, activo, fallos)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8)`,
|
||||
nombre, descripcion, url, categoriaID, paisID, idioma, activo, fallos,
|
||||
)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
if _, rbErr := tx.Exec(context.Background(), "ROLLBACK TO SAVEPOINT sp"); rbErr != nil {
|
||||
errors = append(errors, fmt.Sprintf("Rollback failed for %s: %v", url, rbErr))
|
||||
if err != nil {
|
||||
failed++
|
||||
errors = append(errors, fmt.Sprintf("Error inserting %s: %v", url, err))
|
||||
continue
|
||||
}
|
||||
failed++
|
||||
errors = append(errors, fmt.Sprintf("Error upserting %s: %v", url, err))
|
||||
continue
|
||||
}
|
||||
|
||||
if _, err := tx.Exec(context.Background(), "RELEASE SAVEPOINT sp"); err != nil {
|
||||
failed++
|
||||
errors = append(errors, fmt.Sprintf("Error for %s: %v", url, err))
|
||||
continue
|
||||
}
|
||||
|
||||
imported++
|
||||
|
|
@ -638,7 +515,6 @@ func ImportFeeds(c *gin.Context) {
|
|||
c.JSON(http.StatusInternalServerError, models.ErrorResponse{Error: "Failed to commit transaction", Message: err.Error()})
|
||||
return
|
||||
}
|
||||
committed = true
|
||||
|
||||
c.JSON(http.StatusOK, gin.H{
|
||||
"imported": imported,
|
||||
|
|
@ -649,22 +525,6 @@ func ImportFeeds(c *gin.Context) {
|
|||
})
|
||||
}
|
||||
|
||||
func looksLikeURL(s string) bool {
|
||||
return strings.HasPrefix(s, "http://") || strings.HasPrefix(s, "https://") || strings.HasPrefix(s, "http://www.") || strings.HasPrefix(s, "https://www.") || strings.HasPrefix(s, "www.") || strings.HasPrefix(s, "feed://")
|
||||
}
|
||||
|
||||
func nombreFromURL(u string) string {
|
||||
u = strings.TrimSpace(u)
|
||||
if !strings.HasPrefix(u, "http://") && !strings.HasPrefix(u, "https://") {
|
||||
u = "https://" + u
|
||||
}
|
||||
parsed, err := url.Parse(u)
|
||||
if err != nil || parsed.Host == "" {
|
||||
return u
|
||||
}
|
||||
return strings.TrimPrefix(parsed.Host, "www.")
|
||||
}
|
||||
|
||||
func stringOrEmpty(s *string) string {
|
||||
if s == nil {
|
||||
return ""
|
||||
|
|
|
|||
|
|
@ -1,35 +0,0 @@
|
|||
package handlers
|
||||
|
||||
import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestValidatePageParamsFeed(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
page int
|
||||
perPage int
|
||||
defaultPerPage int
|
||||
maxPerPage int
|
||||
expectedPage int
|
||||
expectedPerPage int
|
||||
}{
|
||||
{"valid", 1, 50, 50, 100, 1, 50},
|
||||
{"page zero", 0, 50, 50, 100, 1, 50},
|
||||
{"per_page too large", 1, 200, 50, 100, 1, 100},
|
||||
{"page negative", -5, 50, 50, 100, 1, 50},
|
||||
{"per_page zero", 1, 0, 50, 100, 1, 50},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
page, perPage := validatePageParams(tt.page, tt.perPage, tt.defaultPerPage, tt.maxPerPage)
|
||||
if page != tt.expectedPage {
|
||||
t.Errorf("page: expected %d, got %d", tt.expectedPage, page)
|
||||
}
|
||||
if perPage != tt.expectedPerPage {
|
||||
t.Errorf("perPage: expected %d, got %d", tt.expectedPerPage, perPage)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
@ -1,54 +1,46 @@
|
|||
package handlers
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/rss2/backend/internal/config"
|
||||
"github.com/rss2/backend/internal/db"
|
||||
"github.com/rss2/backend/internal/models"
|
||||
)
|
||||
|
||||
// getTargetLang returns the target language for translations, defaulting to config DefaultLang
|
||||
func getTargetLang(c *gin.Context) string {
|
||||
lang := c.Query("lang")
|
||||
if lang == "" {
|
||||
cfg := config.Load()
|
||||
lang = cfg.DefaultLang
|
||||
}
|
||||
return lang
|
||||
type NewsResponse struct {
|
||||
ID string `json:"id"`
|
||||
Titulo string `json:"titulo"`
|
||||
Resumen string `json:"resumen"`
|
||||
URL string `json:"url"`
|
||||
Fecha *time.Time `json:"fecha"`
|
||||
ImagenURL *string `json:"imagen_url"`
|
||||
CategoriaID *int64 `json:"categoria_id"`
|
||||
PaisID *int64 `json:"pais_id"`
|
||||
FuenteNombre string `json:"fuente_nombre"`
|
||||
TitleTranslated *string `json:"title_translated"`
|
||||
SummaryTranslated *string `json:"summary_translated"`
|
||||
LangTranslated *string `json:"lang_translated"`
|
||||
Entities []Entity `json:"entities,omitempty"`
|
||||
}
|
||||
|
||||
// GetNews returns paginated news with optional filters
|
||||
// @Summary List news
|
||||
// @Description Returns paginated news with optional filtering by query, category, country, and translation status
|
||||
// @Tags news
|
||||
// @Produce json
|
||||
// @Param page query int false "Page number" default(1) minimum(1)
|
||||
// @Param per_page query int false "Items per page" default(30) minimum(1) maximum(100)
|
||||
// @Param q query string false "Search query"
|
||||
// @Param category_id query string false "Category ID filter"
|
||||
// @Param country_id query string false "Country ID filter"
|
||||
// @Param lang query string false "Target translation language" default(es)
|
||||
// @Param translated_only query string false "Only show translated news" default("false")
|
||||
// @Success 200 {object} map[string]interface{} "news, total, page, per_page, total_pages"
|
||||
// @Failure 500 {object} models.ErrorResponse
|
||||
// @Router /news [get]
|
||||
func GetNews(c *gin.Context) {
|
||||
page, _ := strconv.Atoi(c.DefaultQuery("page", "1"))
|
||||
perPage, _ := strconv.Atoi(c.DefaultQuery("per_page", "30"))
|
||||
page, perPage = validatePageParams(page, perPage, 30, 100)
|
||||
query := c.Query("q")
|
||||
categoryID := c.Query("category_id")
|
||||
countryID := c.Query("country_id")
|
||||
translatedOnly := c.Query("translated_only") == "true"
|
||||
targetLang := getTargetLang(c)
|
||||
|
||||
if page < 1 {
|
||||
page = 1
|
||||
}
|
||||
if perPage < 1 || perPage > 100 {
|
||||
perPage = 30
|
||||
}
|
||||
|
||||
offset := (page - 1) * perPage
|
||||
|
||||
|
|
@ -57,13 +49,7 @@ func GetNews(c *gin.Context) {
|
|||
argNum := 1
|
||||
|
||||
if query != "" {
|
||||
if translatedOnly {
|
||||
// With translated_only, search the translated fields too so users can
|
||||
// find translated news by the target language wording.
|
||||
where += fmt.Sprintf(" AND (n.titulo ILIKE $%d OR n.resumen ILIKE $%d OR t.titulo_trad ILIKE $%d OR t.resumen_trad ILIKE $%d)", argNum, argNum, argNum, argNum)
|
||||
} else {
|
||||
where += fmt.Sprintf(" AND (n.titulo ILIKE $%d OR n.resumen ILIKE $%d)", argNum, argNum)
|
||||
}
|
||||
where += fmt.Sprintf(" AND (n.titulo ILIKE $%d OR n.resumen ILIKE $%d)", argNum, argNum)
|
||||
args = append(args, "%"+query+"%")
|
||||
argNum++
|
||||
}
|
||||
|
|
@ -82,8 +68,7 @@ func GetNews(c *gin.Context) {
|
|||
}
|
||||
|
||||
var total int
|
||||
countQuery := fmt.Sprintf("SELECT COUNT(*) FROM noticias n LEFT JOIN traducciones t ON t.noticia_id = n.id AND t.lang_to = $%d WHERE %s", argNum, where)
|
||||
args = append(args, targetLang)
|
||||
countQuery := fmt.Sprintf("SELECT COUNT(*) FROM noticias n LEFT JOIN traducciones t ON t.noticia_id = n.id AND t.lang_to = 'es' WHERE %s", where)
|
||||
err := db.GetPool().QueryRow(c.Request.Context(), countQuery, args...).Scan(&total)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, models.ErrorResponse{Error: "Failed to count news", Message: err.Error()})
|
||||
|
|
@ -101,18 +86,19 @@ func GetNews(c *gin.Context) {
|
|||
return
|
||||
}
|
||||
|
||||
sqlQuery := `
|
||||
SELECT n.id, n.titulo, COALESCE(n.resumen, ''), n.contenido, n.url, n.fecha, n.imagen_url,
|
||||
n.categoria_id, n.pais_id, n.fuente_nombre, n.lang,
|
||||
sqlQuery := fmt.Sprintf(`
|
||||
SELECT n.id, n.titulo, COALESCE(n.resumen, ''), n.url, n.fecha, n.imagen_url,
|
||||
n.categoria_id, n.pais_id, n.fuente_nombre,
|
||||
t.titulo_trad,
|
||||
t.resumen_trad,
|
||||
t.lang_to as lang_trad
|
||||
FROM noticias n
|
||||
LEFT JOIN traducciones t ON t.noticia_id = n.id AND t.lang_to = $` + strconv.Itoa(argNum) + `
|
||||
WHERE ` + where + `
|
||||
ORDER BY n.fecha DESC LIMIT $` + strconv.Itoa(argNum+1) + ` OFFSET $` + strconv.Itoa(argNum+2)
|
||||
LEFT JOIN traducciones t ON t.noticia_id = n.id AND t.lang_to = 'es'
|
||||
WHERE %s
|
||||
ORDER BY n.fecha DESC LIMIT $%d OFFSET $%d
|
||||
`, where, argNum, argNum+1)
|
||||
|
||||
args = append(args, targetLang, perPage, offset)
|
||||
args = append(args, perPage, offset)
|
||||
|
||||
rows, err := db.GetPool().Query(c.Request.Context(), sqlQuery, args...)
|
||||
if err != nil {
|
||||
|
|
@ -121,16 +107,15 @@ func GetNews(c *gin.Context) {
|
|||
}
|
||||
defer rows.Close()
|
||||
|
||||
var newsList []models.NewsWithTranslations
|
||||
var newsList []NewsResponse
|
||||
for rows.Next() {
|
||||
var n models.NewsWithTranslations
|
||||
var n NewsResponse
|
||||
var imagenURL, fuenteNombre *string
|
||||
var categoriaID, paisID *int32
|
||||
var lang string
|
||||
|
||||
err := rows.Scan(
|
||||
&n.ID, &n.Titulo, &n.Resumen, &n.Contenido, &n.URL, &n.Fecha, &imagenURL,
|
||||
&categoriaID, &paisID, &fuenteNombre, &lang,
|
||||
&n.ID, &n.Titulo, &n.Resumen, &n.URL, &n.Fecha, &imagenURL,
|
||||
&categoriaID, &paisID, &fuenteNombre,
|
||||
&n.TitleTranslated, &n.SummaryTranslated, &n.LangTranslated,
|
||||
)
|
||||
if err != nil {
|
||||
|
|
@ -144,13 +129,12 @@ func GetNews(c *gin.Context) {
|
|||
}
|
||||
if categoriaID != nil {
|
||||
catID := int64(*categoriaID)
|
||||
n.CategoryID = &catID
|
||||
n.CategoriaID = &catID
|
||||
}
|
||||
if paisID != nil {
|
||||
pID := int64(*paisID)
|
||||
n.CountryID = &pID
|
||||
n.PaisID = &pID
|
||||
}
|
||||
n.Lang = lang
|
||||
newsList = append(newsList, n)
|
||||
}
|
||||
|
||||
|
|
@ -165,169 +149,27 @@ func GetNews(c *gin.Context) {
|
|||
})
|
||||
}
|
||||
|
||||
// GetEntityNews returns the (translated) news that mention a given entity/alias value.
|
||||
func GetEntityNews(c *gin.Context) {
|
||||
valor := c.Query("valor")
|
||||
tipo := c.DefaultQuery("tipo", "persona")
|
||||
page, _ := strconv.Atoi(c.DefaultQuery("page", "1"))
|
||||
perPage, _ := strconv.Atoi(c.DefaultQuery("per_page", "20"))
|
||||
page, perPage = validatePageParams(page, perPage, 20, 50)
|
||||
offset := (page - 1) * perPage
|
||||
targetLang := getTargetLang(c)
|
||||
|
||||
daysStr := c.Query("days")
|
||||
days := 0
|
||||
if daysStr != "" {
|
||||
days, _ = strconv.Atoi(daysStr)
|
||||
if days < 1 {
|
||||
days = 0
|
||||
}
|
||||
}
|
||||
today := c.Query("today") == "true"
|
||||
offsetStr := c.Query("day_offset")
|
||||
dayOffset := 0
|
||||
useDayOffset := false
|
||||
if offsetStr != "" {
|
||||
dayOffset, _ = strconv.Atoi(offsetStr)
|
||||
if dayOffset < 0 {
|
||||
dayOffset = 0
|
||||
}
|
||||
useDayOffset = true
|
||||
}
|
||||
|
||||
params := []interface{}{valor, tipo, targetLang}
|
||||
next := 4
|
||||
timeCond := ""
|
||||
switch {
|
||||
case useDayOffset:
|
||||
// Día exacto: desde el inicio del día (hoy - offset) hasta el inicio del día siguiente
|
||||
timeCond = fmt.Sprintf(
|
||||
" AND n.fecha >= (CURRENT_DATE - %d)::timestamp AND n.fecha < (CURRENT_DATE - %d + 1)::timestamp",
|
||||
dayOffset, dayOffset)
|
||||
case today:
|
||||
timeCond = " AND n.fecha >= date_trunc('day', NOW())"
|
||||
case days > 0:
|
||||
timeCond = fmt.Sprintf(" AND n.fecha >= NOW() - make_interval(days => $%d)", next)
|
||||
params = append(params, days)
|
||||
next++
|
||||
}
|
||||
|
||||
base := fmt.Sprintf(`
|
||||
FROM tags_noticia tn
|
||||
JOIN tags t ON tn.tag_id = t.id
|
||||
JOIN traducciones tr ON tn.traduccion_id = tr.id AND tr.lang_to = $3
|
||||
JOIN noticias n ON tr.noticia_id = n.id
|
||||
LEFT JOIN entity_aliases ea ON LOWER(ea.alias) = LOWER(t.valor) AND ea.tipo = t.tipo
|
||||
WHERE LOWER(COALESCE(ea.canonical_name, t.valor)) = LOWER($1) AND t.tipo = $2%s`, timeCond)
|
||||
|
||||
var total int
|
||||
if err := db.GetPool().QueryRow(c.Request.Context(),
|
||||
fmt.Sprintf("SELECT COUNT(DISTINCT tn.noticia_id) %s", base), params...).Scan(&total); err != nil {
|
||||
c.JSON(http.StatusInternalServerError, models.ErrorResponse{Error: "Failed to count entity news", Message: err.Error()})
|
||||
return
|
||||
}
|
||||
|
||||
query := fmt.Sprintf(`
|
||||
SELECT DISTINCT n.id, n.titulo, COALESCE(n.resumen,''), n.contenido, n.url, n.fecha, n.imagen_url,
|
||||
n.fuente_nombre, n.lang, tr.titulo_trad, tr.resumen_trad
|
||||
%s
|
||||
ORDER BY n.fecha DESC
|
||||
LIMIT $%d OFFSET $%d`, base, next, next+1)
|
||||
|
||||
rows, err := db.GetPool().Query(c.Request.Context(), query, append(params, perPage, offset)...)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, models.ErrorResponse{Error: "Failed to get entity news", Message: err.Error()})
|
||||
return
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var news []models.NewsWithTranslations
|
||||
for rows.Next() {
|
||||
var n models.NewsWithTranslations
|
||||
var img *string
|
||||
if err := rows.Scan(&n.ID, &n.Titulo, &n.Resumen, &n.Contenido, &n.URL, &n.Fecha, &img,
|
||||
&n.FuenteNombre, &n.Lang, &n.TitleTranslated, &n.SummaryTranslated); err != nil {
|
||||
continue
|
||||
}
|
||||
if img != nil {
|
||||
n.ImagenURL = img
|
||||
}
|
||||
news = append(news, n)
|
||||
}
|
||||
if news == nil {
|
||||
news = []models.NewsWithTranslations{}
|
||||
}
|
||||
|
||||
totalPages := (total + perPage - 1) / perPage
|
||||
|
||||
c.JSON(http.StatusOK, gin.H{
|
||||
"news": news,
|
||||
"total": total,
|
||||
"page": page,
|
||||
"per_page": perPage,
|
||||
"total_pages": totalPages,
|
||||
})
|
||||
}
|
||||
|
||||
// GetNewsByID returns a single news item with entities
|
||||
// @Summary Get news by ID
|
||||
// @Description Returns a single news item with translations and associated entities
|
||||
// @Tags news
|
||||
// @Produce json
|
||||
// @Param id path string true "News ID"
|
||||
// @Param lang query string false "Target translation language" default(es)
|
||||
// @Success 200 {object} models.NewsWithTranslations
|
||||
// @Failure 404 {object} models.ErrorResponse
|
||||
// @Failure 500 {object} models.ErrorResponse
|
||||
// @Router /news/{id} [get]
|
||||
func GetNewsByID(c *gin.Context) {
|
||||
id := c.Param("id")
|
||||
targetLang := getTargetLang(c)
|
||||
|
||||
// Combined query to fetch news with entities in one round trip
|
||||
sqlQuery := `
|
||||
SELECT
|
||||
n.id, n.titulo, COALESCE(n.resumen, ''), n.contenido, n.url, n.fecha, n.imagen_url,
|
||||
n.categoria_id, n.pais_id, n.fuente_nombre, n.lang,
|
||||
SELECT n.id, n.titulo, COALESCE(n.resumen, ''), n.url, n.fecha, n.imagen_url,
|
||||
n.categoria_id, n.pais_id, n.fuente_nombre,
|
||||
t.titulo_trad,
|
||||
t.resumen_trad,
|
||||
t.lang_to as lang_trad,
|
||||
json_agg(
|
||||
json_build_object(
|
||||
'valor', ent.valor,
|
||||
'tipo', ent.tipo,
|
||||
'count', ent.cnt,
|
||||
'wiki_summary', ent.wiki_summary,
|
||||
'wiki_url', ent.wiki_url,
|
||||
'image_path', ent.image_path
|
||||
)
|
||||
) FILTER (WHERE ent.valor IS NOT NULL) as entities
|
||||
t.lang_to as lang_trad
|
||||
FROM noticias n
|
||||
LEFT JOIN traducciones t ON t.noticia_id = n.id AND t.lang_to = $1
|
||||
LEFT JOIN (
|
||||
SELECT tn.noticia_id, t.valor, t.tipo, 1 as cnt, t.wiki_summary, t.wiki_url, t.image_path
|
||||
FROM tags_noticia tn
|
||||
JOIN tags t ON tn.tag_id = t.id
|
||||
JOIN traducciones tr ON tn.traduccion_id = tr.id
|
||||
WHERE t.tipo IN ('persona', 'organizacion')
|
||||
) ent ON ent.noticia_id = n.id
|
||||
WHERE n.id = $2
|
||||
GROUP BY n.id, n.titulo, n.resumen, n.contenido, n.url, n.fecha, n.imagen_url,
|
||||
n.categoria_id, n.pais_id, n.fuente_nombre, n.lang,
|
||||
t.titulo_trad, t.resumen_trad, t.lang_to
|
||||
`
|
||||
LEFT JOIN traducciones t ON t.noticia_id = n.id AND t.lang_to = 'es'
|
||||
WHERE n.id = $1`
|
||||
|
||||
var n models.NewsWithTranslations
|
||||
var n NewsResponse
|
||||
var imagenURL, fuenteNombre *string
|
||||
var categoriaID, paisID *int32
|
||||
var lang string
|
||||
var entitiesJSON *string
|
||||
|
||||
err := db.GetPool().QueryRow(c.Request.Context(), sqlQuery, targetLang, id).Scan(
|
||||
&n.ID, &n.Titulo, &n.Resumen, &n.Contenido, &n.URL, &n.Fecha, &imagenURL,
|
||||
&categoriaID, &paisID, &fuenteNombre, &lang,
|
||||
err := db.GetPool().QueryRow(c.Request.Context(), sqlQuery, id).Scan(
|
||||
&n.ID, &n.Titulo, &n.Resumen, &n.URL, &n.Fecha, &imagenURL,
|
||||
&categoriaID, &paisID, &fuenteNombre,
|
||||
&n.TitleTranslated, &n.SummaryTranslated, &n.LangTranslated,
|
||||
&entitiesJSON,
|
||||
)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusNotFound, models.ErrorResponse{Error: "News not found"})
|
||||
|
|
@ -342,23 +184,36 @@ func GetNewsByID(c *gin.Context) {
|
|||
}
|
||||
if categoriaID != nil {
|
||||
catID := int64(*categoriaID)
|
||||
n.CategoryID = &catID
|
||||
n.CategoriaID = &catID
|
||||
}
|
||||
if paisID != nil {
|
||||
pID := int64(*paisID)
|
||||
n.CountryID = &pID
|
||||
n.PaisID = &pID
|
||||
}
|
||||
n.Lang = lang
|
||||
|
||||
// Parse entities from JSON
|
||||
if entitiesJSON != nil && *entitiesJSON != "null" {
|
||||
var entities []models.Entity
|
||||
if err := json.Unmarshal([]byte(*entitiesJSON), &entities); err == nil {
|
||||
n.Entities = entities
|
||||
// Fetch entities for this news
|
||||
entitiesQuery := `
|
||||
SELECT t.valor, t.tipo, 1 as cnt, t.wiki_summary, t.wiki_url, t.image_path
|
||||
FROM tags_noticia tn
|
||||
JOIN tags t ON tn.tag_id = t.id
|
||||
JOIN traducciones tr ON tn.traduccion_id = tr.id
|
||||
WHERE tr.noticia_id = $1 AND t.tipo IN ('persona', 'organizacion')
|
||||
`
|
||||
rows, err := db.GetPool().Query(c.Request.Context(), entitiesQuery, id)
|
||||
var entities []Entity
|
||||
if err == nil {
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
var e Entity
|
||||
if err := rows.Scan(&e.Valor, &e.Tipo, &e.Count, &e.WikiSummary, &e.WikiURL, &e.ImagePath); err == nil {
|
||||
entities = append(entities, e)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
n.Entities = []models.Entity{}
|
||||
}
|
||||
if entities == nil {
|
||||
entities = []Entity{}
|
||||
}
|
||||
n.Entities = entities
|
||||
|
||||
c.JSON(http.StatusOK, n)
|
||||
}
|
||||
|
|
@ -380,20 +235,23 @@ func DeleteNews(c *gin.Context) {
|
|||
c.JSON(http.StatusOK, models.SuccessResponse{Message: "News deleted successfully"})
|
||||
}
|
||||
|
||||
// GetEntities returns paginated entities with optional filters
|
||||
// @Summary List entities
|
||||
// @Description Returns paginated entities (personas, organizations, locations) with optional filters
|
||||
// @Tags entities
|
||||
// @Produce json
|
||||
// @Param tipo query string false "Entity type" default("persona") Enums(persona, organizacion, lugar, tema)
|
||||
// @Param page query int false "Page number" default(1) minimum(1)
|
||||
// @Param per_page query int false "Items per page" default(50) minimum(1) maximum(100)
|
||||
// @Param country_id query string false "Country ID filter"
|
||||
// @Param category_id query string false "Category ID filter"
|
||||
// @Param q query string false "Search query"
|
||||
// @Success 200 {object} models.EntityListResponse
|
||||
// @Failure 500 {object} models.ErrorResponse
|
||||
// @Router /entities [get]
|
||||
type Entity struct {
|
||||
Valor string `json:"valor"`
|
||||
Tipo string `json:"tipo"`
|
||||
Count int `json:"count"`
|
||||
WikiSummary *string `json:"wiki_summary"`
|
||||
WikiURL *string `json:"wiki_url"`
|
||||
ImagePath *string `json:"image_path"`
|
||||
}
|
||||
|
||||
type EntityListResponse struct {
|
||||
Entities []Entity `json:"entities"`
|
||||
Total int `json:"total"`
|
||||
Page int `json:"page"`
|
||||
PerPage int `json:"per_page"`
|
||||
TotalPages int `json:"total_pages"`
|
||||
}
|
||||
|
||||
func GetEntities(c *gin.Context) {
|
||||
countryID := c.Query("country_id")
|
||||
categoryID := c.Query("category_id")
|
||||
|
|
@ -401,9 +259,18 @@ func GetEntities(c *gin.Context) {
|
|||
|
||||
q := c.Query("q")
|
||||
|
||||
page, _ := strconv.Atoi(c.DefaultQuery("page", "1"))
|
||||
perPage, _ := strconv.Atoi(c.DefaultQuery("per_page", "50"))
|
||||
page, perPage = validatePageParams(page, perPage, 50, 100)
|
||||
pageStr := c.DefaultQuery("page", "1")
|
||||
perPageStr := c.DefaultQuery("per_page", "50")
|
||||
|
||||
page, _ := strconv.Atoi(pageStr)
|
||||
perPage, _ := strconv.Atoi(perPageStr)
|
||||
|
||||
if page < 1 {
|
||||
page = 1
|
||||
}
|
||||
if perPage < 1 || perPage > 100 {
|
||||
perPage = 50
|
||||
}
|
||||
|
||||
offset := (page - 1) * perPage
|
||||
|
||||
|
|
@ -444,8 +311,8 @@ func GetEntities(c *gin.Context) {
|
|||
}
|
||||
|
||||
if total == 0 {
|
||||
c.JSON(http.StatusOK, models.EntityListResponse{
|
||||
Entities: []models.Entity{},
|
||||
c.JSON(http.StatusOK, EntityListResponse{
|
||||
Entities: []Entity{},
|
||||
Total: 0,
|
||||
Page: page,
|
||||
PerPage: perPage,
|
||||
|
|
@ -477,9 +344,9 @@ func GetEntities(c *gin.Context) {
|
|||
}
|
||||
defer rows.Close()
|
||||
|
||||
var entities []models.Entity
|
||||
var entities []Entity
|
||||
for rows.Next() {
|
||||
var e models.Entity
|
||||
var e Entity
|
||||
if err := rows.Scan(&e.Valor, &e.Tipo, &e.Count, &e.WikiSummary, &e.WikiURL, &e.ImagePath); err != nil {
|
||||
continue
|
||||
}
|
||||
|
|
@ -487,12 +354,12 @@ func GetEntities(c *gin.Context) {
|
|||
}
|
||||
|
||||
if entities == nil {
|
||||
entities = []models.Entity{}
|
||||
entities = []Entity{}
|
||||
}
|
||||
|
||||
totalPages := (total + perPage - 1) / perPage
|
||||
|
||||
c.JSON(http.StatusOK, models.EntityListResponse{
|
||||
c.JSON(http.StatusOK, EntityListResponse{
|
||||
Entities: entities,
|
||||
Total: total,
|
||||
Page: page,
|
||||
|
|
@ -500,123 +367,3 @@ func GetEntities(c *gin.Context) {
|
|||
TotalPages: totalPages,
|
||||
})
|
||||
}
|
||||
|
||||
func GetEntityMentions(c *gin.Context) {
|
||||
valuesRaw := c.DefaultQuery("values", "")
|
||||
days := 30
|
||||
if v := c.Query("days"); v != "" {
|
||||
if d, err := strconv.Atoi(v); err == nil && d >= 1 && d <= 365 {
|
||||
days = d
|
||||
}
|
||||
}
|
||||
|
||||
var values []string
|
||||
for _, v := range strings.Split(valuesRaw, ",") {
|
||||
v = strings.TrimSpace(v)
|
||||
if v != "" {
|
||||
values = append(values, v)
|
||||
}
|
||||
}
|
||||
if len(values) == 0 {
|
||||
c.JSON(http.StatusBadRequest, models.ErrorResponse{Error: "values is required (comma separated)"})
|
||||
return
|
||||
}
|
||||
if len(values) > 20 {
|
||||
c.JSON(http.StatusBadRequest, models.ErrorResponse{Error: "Too many values (max 20)"})
|
||||
return
|
||||
}
|
||||
|
||||
start := time.Now().AddDate(0, 0, -(days - 1)).Truncate(24 * time.Hour)
|
||||
|
||||
series := make([]models.MentionSeries, 0, len(values))
|
||||
for _, valor := range values {
|
||||
s, err := buildMentionSeries(c.Request.Context(), valor, days)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, models.ErrorResponse{Error: "Failed to get mentions", Message: err.Error()})
|
||||
return
|
||||
}
|
||||
|
||||
m := map[string]int{}
|
||||
for _, p := range s.Data {
|
||||
m[p.Fecha] = p.Count
|
||||
}
|
||||
data := make([]models.MentionPoint, 0, days)
|
||||
for i := 0; i < days; i++ {
|
||||
fecha := start.AddDate(0, 0, i).Format("2006-01-02")
|
||||
data = append(data, models.MentionPoint{Fecha: fecha, Count: m[fecha]})
|
||||
}
|
||||
s.Data = data
|
||||
s.Count = 0
|
||||
for _, p := range s.Data {
|
||||
s.Count += p.Count
|
||||
}
|
||||
if s.Tipo == "" {
|
||||
s.Tipo = "desconocido"
|
||||
}
|
||||
series = append(series, *s)
|
||||
}
|
||||
|
||||
c.JSON(http.StatusOK, models.MentionsResponse{Days: days, Series: series})
|
||||
}
|
||||
|
||||
func buildMentionSeries(ctx context.Context, valor string, days int) (*models.MentionSeries, error) {
|
||||
s := &models.MentionSeries{Valor: valor}
|
||||
|
||||
var tagIDs []int64
|
||||
rows, err := db.GetPool().Query(ctx, `
|
||||
SELECT DISTINCT t.id, t.tipo
|
||||
FROM tags t
|
||||
LEFT JOIN entity_aliases ea ON ea.tipo = t.tipo AND LOWER(ea.alias) = LOWER(t.valor)
|
||||
WHERE LOWER(t.valor) = LOWER($1)
|
||||
OR LOWER(COALESCE(ea.canonical_name, t.valor)) = LOWER($1)
|
||||
`, valor)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for rows.Next() {
|
||||
var id int64
|
||||
var tipo string
|
||||
if err := rows.Scan(&id, &tipo); err != nil {
|
||||
continue
|
||||
}
|
||||
tagIDs = append(tagIDs, id)
|
||||
if s.Tipo == "" {
|
||||
s.Tipo = tipo
|
||||
}
|
||||
}
|
||||
rows.Close()
|
||||
|
||||
if len(tagIDs) == 0 {
|
||||
s.Data = []models.MentionPoint{}
|
||||
return s, nil
|
||||
}
|
||||
|
||||
drows, err := db.GetPool().Query(ctx, `
|
||||
SELECT n.fecha::date::text AS dia, COUNT(DISTINCT n.id) AS cnt
|
||||
FROM tags_noticia tn
|
||||
JOIN tags t ON tn.tag_id = t.id
|
||||
JOIN traducciones tr ON tn.traduccion_id = tr.id
|
||||
JOIN noticias n ON tr.noticia_id = n.id
|
||||
WHERE t.id = ANY($1) AND n.fecha >= CURRENT_DATE - ($2::int - 1)
|
||||
GROUP BY n.fecha::date
|
||||
ORDER BY n.fecha::date
|
||||
`, tagIDs, days)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer drows.Close()
|
||||
|
||||
for drows.Next() {
|
||||
var fecha string
|
||||
var cnt int
|
||||
if err := drows.Scan(&fecha, &cnt); err != nil {
|
||||
continue
|
||||
}
|
||||
s.Data = append(s.Data, models.MentionPoint{Fecha: fecha, Count: cnt})
|
||||
}
|
||||
if s.Data == nil {
|
||||
s.Data = []models.MentionPoint{}
|
||||
}
|
||||
|
||||
return s, nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,38 +0,0 @@
|
|||
package handlers
|
||||
|
||||
import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestValidatePageParams(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
page int
|
||||
perPage int
|
||||
defaultPerPage int
|
||||
maxPerPage int
|
||||
expectedPage int
|
||||
expectedPerPage int
|
||||
}{
|
||||
{"normal", 1, 30, 30, 100, 1, 30},
|
||||
{"page zero", 0, 30, 30, 100, 1, 30},
|
||||
{"page negative", -5, 30, 30, 100, 1, 30},
|
||||
{"per_page zero", 1, 0, 30, 100, 1, 30},
|
||||
{"per_page negative", 1, -10, 30, 100, 1, 30},
|
||||
{"per_page exceeds max", 1, 200, 30, 100, 1, 100},
|
||||
{"page exceeds max", 999, 30, 30, 100, 999, 30},
|
||||
{"custom max", 1, 200, 20, 50, 1, 50},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
page, perPage := validatePageParams(tt.page, tt.perPage, tt.defaultPerPage, tt.maxPerPage)
|
||||
if page != tt.expectedPage {
|
||||
t.Errorf("page: expected %d, got %d", tt.expectedPage, page)
|
||||
}
|
||||
if perPage != tt.expectedPerPage {
|
||||
t.Errorf("perPage: expected %d, got %d", tt.expectedPerPage, perPage)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
@ -1,187 +0,0 @@
|
|||
package handlers
|
||||
|
||||
import (
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"log"
|
||||
"net/http"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/rss2/backend/internal/db"
|
||||
"github.com/rss2/backend/internal/models"
|
||||
)
|
||||
|
||||
func CreateRemoteWorker(c *gin.Context) {
|
||||
var req struct {
|
||||
Name string `json:"name" binding:"required"`
|
||||
Capabilities string `json:"capabilities"`
|
||||
}
|
||||
|
||||
if err := c.ShouldBindJSON(&req); err != nil {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": "Invalid request", "message": err.Error()})
|
||||
return
|
||||
}
|
||||
|
||||
if req.Capabilities == "" {
|
||||
req.Capabilities = "cpu"
|
||||
}
|
||||
|
||||
apiKey := generateAPIKey()
|
||||
|
||||
ctx := c.Request.Context()
|
||||
var workerID int
|
||||
err := db.GetPool().QueryRow(ctx, `
|
||||
INSERT INTO remote_workers (name, api_key, capabilities, status, created_at)
|
||||
VALUES ($1, $2, $3, 'offline', NOW())
|
||||
RETURNING id
|
||||
`, req.Name, apiKey, req.Capabilities).Scan(&workerID)
|
||||
|
||||
if err != nil {
|
||||
log.Printf("Error creating remote worker: %v", err)
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to create worker"})
|
||||
return
|
||||
}
|
||||
|
||||
c.JSON(http.StatusCreated, gin.H{
|
||||
"id": workerID,
|
||||
"name": req.Name,
|
||||
"api_key": apiKey,
|
||||
"capabilities": req.Capabilities,
|
||||
"status": "offline",
|
||||
})
|
||||
}
|
||||
|
||||
func ListRemoteWorkers(c *gin.Context) {
|
||||
ctx := c.Request.Context()
|
||||
|
||||
rows, err := db.GetPool().Query(ctx, `
|
||||
SELECT id, name, api_key, capabilities, status, last_seen, created_at
|
||||
FROM remote_workers
|
||||
ORDER BY created_at DESC
|
||||
`)
|
||||
if err != nil {
|
||||
log.Printf("Error listing remote workers: %v", err)
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to list workers"})
|
||||
return
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var workers []models.RemoteWorker
|
||||
for rows.Next() {
|
||||
var w models.RemoteWorker
|
||||
if err := rows.Scan(&w.ID, &w.Name, &w.APIKey, &w.Capabilities, &w.Status, &w.LastSeen, &w.CreatedAt); err != nil {
|
||||
log.Printf("Error scanning worker: %v", err)
|
||||
continue
|
||||
}
|
||||
workers = append(workers, w)
|
||||
}
|
||||
|
||||
if workers == nil {
|
||||
workers = []models.RemoteWorker{}
|
||||
}
|
||||
|
||||
c.JSON(http.StatusOK, workers)
|
||||
}
|
||||
|
||||
func GetRemoteWorker(c *gin.Context) {
|
||||
id := c.Param("id")
|
||||
ctx := c.Request.Context()
|
||||
|
||||
var w models.RemoteWorker
|
||||
err := db.GetPool().QueryRow(ctx, `
|
||||
SELECT id, name, api_key, capabilities, status, last_seen, created_at
|
||||
FROM remote_workers
|
||||
WHERE id = $1
|
||||
`, id).Scan(&w.ID, &w.Name, &w.APIKey, &w.Capabilities, &w.Status, &w.LastSeen, &w.CreatedAt)
|
||||
|
||||
if err != nil {
|
||||
c.JSON(http.StatusNotFound, gin.H{"error": "Worker not found"})
|
||||
return
|
||||
}
|
||||
|
||||
var stats struct {
|
||||
JobsCompleted int `json:"jobs_completed"`
|
||||
JobsPending int `json:"jobs_pending"`
|
||||
}
|
||||
|
||||
db.GetPool().QueryRow(ctx, `
|
||||
SELECT COUNT(*) FROM traducciones WHERE worker_id = $1 AND status = 'done'
|
||||
`, id).Scan(&stats.JobsCompleted)
|
||||
|
||||
db.GetPool().QueryRow(ctx, `
|
||||
SELECT COUNT(*) FROM traducciones WHERE worker_id = $1 AND status = 'pending'
|
||||
`, id).Scan(&stats.JobsPending)
|
||||
|
||||
c.JSON(http.StatusOK, gin.H{
|
||||
"worker": w,
|
||||
"stats": stats,
|
||||
})
|
||||
}
|
||||
|
||||
func DeleteRemoteWorker(c *gin.Context) {
|
||||
id := c.Param("id")
|
||||
ctx := c.Request.Context()
|
||||
|
||||
result, err := db.GetPool().Exec(ctx, `
|
||||
DELETE FROM remote_workers WHERE id = $1
|
||||
`, id)
|
||||
|
||||
if err != nil {
|
||||
log.Printf("Error deleting remote worker: %v", err)
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to delete worker"})
|
||||
return
|
||||
}
|
||||
|
||||
if result.RowsAffected() == 0 {
|
||||
c.JSON(http.StatusNotFound, gin.H{"error": "Worker not found"})
|
||||
return
|
||||
}
|
||||
|
||||
c.JSON(http.StatusOK, gin.H{"message": "Worker deleted"})
|
||||
}
|
||||
|
||||
func ToggleRemoteWorker(c *gin.Context) {
|
||||
id := c.Param("id")
|
||||
ctx := c.Request.Context()
|
||||
|
||||
var currentStatus string
|
||||
err := db.GetPool().QueryRow(ctx, `SELECT status FROM remote_workers WHERE id = $1`, id).Scan(¤tStatus)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusNotFound, gin.H{"error": "Worker not found"})
|
||||
return
|
||||
}
|
||||
|
||||
newStatus := "disabled"
|
||||
if currentStatus == "disabled" {
|
||||
newStatus = "offline"
|
||||
}
|
||||
|
||||
_, err = db.GetPool().Exec(ctx, `UPDATE remote_workers SET status = $1 WHERE id = $2`, newStatus, id)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to update worker status"})
|
||||
return
|
||||
}
|
||||
|
||||
c.JSON(http.StatusOK, gin.H{"status": newStatus})
|
||||
}
|
||||
|
||||
func RegenerateAPIKey(c *gin.Context) {
|
||||
id := c.Param("id")
|
||||
ctx := c.Request.Context()
|
||||
|
||||
newAPIKey := generateAPIKey()
|
||||
|
||||
_, err := db.GetPool().Exec(ctx, `UPDATE remote_workers SET api_key = $1 WHERE id = $2`, newAPIKey, id)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to regenerate API key"})
|
||||
return
|
||||
}
|
||||
|
||||
c.JSON(http.StatusOK, gin.H{"api_key": newAPIKey})
|
||||
}
|
||||
|
||||
func generateAPIKey() string {
|
||||
bytes := make([]byte, 32)
|
||||
rand.Read(bytes)
|
||||
return hex.EncodeToString(bytes)
|
||||
}
|
||||
|
|
@ -1,107 +1,19 @@
|
|||
package handlers
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/rss2/backend/internal/auth"
|
||||
"github.com/rss2/backend/internal/cache"
|
||||
"github.com/rss2/backend/internal/db"
|
||||
"github.com/rss2/backend/internal/models"
|
||||
"github.com/rss2/backend/internal/services"
|
||||
)
|
||||
|
||||
// LogSearch records a logged-in user's search term so frequent searches gain priority.
|
||||
func LogSearch(c *gin.Context) {
|
||||
user := c.MustGet("user").(*auth.Claims)
|
||||
var req struct {
|
||||
Query string `json:"q"`
|
||||
}
|
||||
if err := c.ShouldBindJSON(&req); err != nil {
|
||||
c.JSON(http.StatusBadRequest, models.ErrorResponse{Error: "Invalid request"})
|
||||
return
|
||||
}
|
||||
term := strings.ToLower(strings.TrimSpace(req.Query))
|
||||
if term == "" {
|
||||
c.JSON(http.StatusBadRequest, models.ErrorResponse{Error: "q requerido"})
|
||||
return
|
||||
}
|
||||
if len(term) > 100 {
|
||||
term = term[:100]
|
||||
}
|
||||
|
||||
_, err := db.GetPool().Exec(c.Request.Context(), `
|
||||
INSERT INTO user_search_tags (user_id, term, count, last_used)
|
||||
VALUES ($1, $2, 1, NOW())
|
||||
ON CONFLICT (user_id, term)
|
||||
DO UPDATE SET count = user_search_tags.count + 1, last_used = NOW()
|
||||
`, user.UserID, term)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, models.ErrorResponse{Error: "Failed to save search", Message: err.Error()})
|
||||
return
|
||||
}
|
||||
|
||||
c.JSON(http.StatusOK, gin.H{"ok": true})
|
||||
}
|
||||
|
||||
// SearchSuggestions returns the user's most used search terms (dynamic tags).
|
||||
func SearchSuggestions(c *gin.Context) {
|
||||
user := c.MustGet("user").(*auth.Claims)
|
||||
q := strings.TrimSpace(c.Query("q"))
|
||||
|
||||
query := `
|
||||
SELECT term FROM user_search_tags
|
||||
WHERE user_id = $1`
|
||||
args := []interface{}{user.UserID}
|
||||
if q != "" {
|
||||
query += ` AND term ILIKE $2`
|
||||
args = append(args, "%"+q+"%")
|
||||
}
|
||||
query += ` ORDER BY count DESC, last_used DESC LIMIT 10`
|
||||
|
||||
rows, err := db.GetPool().Query(c.Request.Context(), query, args...)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, models.ErrorResponse{Error: "Failed to get suggestions", Message: err.Error()})
|
||||
return
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
terms := []string{}
|
||||
for rows.Next() {
|
||||
var t string
|
||||
if err := rows.Scan(&t); err != nil {
|
||||
continue
|
||||
}
|
||||
terms = append(terms, t)
|
||||
}
|
||||
|
||||
c.JSON(http.StatusOK, gin.H{"terms": terms})
|
||||
}
|
||||
|
||||
// SearchNews searches news with various filters
|
||||
// @Summary Search news
|
||||
// @Description Search news with text query, language, category, country, and semantic search options
|
||||
// @Tags search
|
||||
// @Produce json
|
||||
// @Param q query string false "Search query"
|
||||
// @Param page query int false "Page number" default(1) minimum(1)
|
||||
// @Param per_page query int false "Items per page" default(30) minimum(1) maximum(100)
|
||||
// @Param lang query string false "Language code" default("es")
|
||||
// @Param categoria_id query string false "Category ID filter"
|
||||
// @Param pais_id query string false "Country ID filter"
|
||||
// @Param semantic query string false "Use semantic search" default("false")
|
||||
// @Success 200 {object} map[string]interface{} "news, total, page, per_page, total_pages"
|
||||
// @Failure 400 {object} models.ErrorResponse
|
||||
// @Failure 500 {object} models.ErrorResponse
|
||||
// @Router /search [get]
|
||||
func SearchNews(c *gin.Context) {
|
||||
query := c.Query("q")
|
||||
page, _ := strconv.Atoi(c.DefaultQuery("page", "1"))
|
||||
perPage, _ := strconv.Atoi(c.DefaultQuery("per_page", "30"))
|
||||
page, perPage = validatePageParams(page, perPage, 30, 100)
|
||||
lang := c.DefaultQuery("lang", "")
|
||||
categoriaID := c.Query("categoria_id")
|
||||
paisID := c.Query("pais_id")
|
||||
|
|
@ -112,6 +24,13 @@ func SearchNews(c *gin.Context) {
|
|||
return
|
||||
}
|
||||
|
||||
if page < 1 {
|
||||
page = 1
|
||||
}
|
||||
if perPage < 1 || perPage > 100 {
|
||||
perPage = 30
|
||||
}
|
||||
|
||||
// Default to Spanish if no lang specified
|
||||
if lang == "" {
|
||||
lang = "es"
|
||||
|
|
@ -129,58 +48,59 @@ func SearchNews(c *gin.Context) {
|
|||
return
|
||||
}
|
||||
|
||||
// Cache key for search results
|
||||
cacheKey := cache.SearchKey(query, lang, page, perPage)
|
||||
|
||||
// Try to get from cache
|
||||
if cached, err := cache.Get(ctx, cacheKey); err == nil && cached != "" {
|
||||
c.Data(http.StatusOK, "application/json", []byte(cached))
|
||||
return
|
||||
}
|
||||
|
||||
offset := (page - 1) * perPage
|
||||
|
||||
// Build dynamic query
|
||||
args := []interface{}{lang}
|
||||
argNum := 2
|
||||
args := []interface{}{}
|
||||
argNum := 1
|
||||
whereClause := "WHERE 1=1"
|
||||
|
||||
if query != "" {
|
||||
whereClause += " AND (n.titulo ILIKE $" + strconv.Itoa(argNum) + " OR n.resumen ILIKE $" + strconv.Itoa(argNum) + ")"
|
||||
whereClause += " AND (n.titulo ILIKE $" + strconv.Itoa(argNum) + " OR n.resumen ILIKE $" + strconv.Itoa(argNum) + " OR n.contenido ILIKE $" + strconv.Itoa(argNum) + ")"
|
||||
args = append(args, "%"+query+"%")
|
||||
argNum++
|
||||
}
|
||||
|
||||
if lang != "" {
|
||||
whereClause += " AND t.lang_to = $" + strconv.Itoa(argNum)
|
||||
args = append(args, lang)
|
||||
argNum++
|
||||
}
|
||||
|
||||
if categoriaID != "" {
|
||||
if catID, err := strconv.ParseInt(categoriaID, 10, 64); err == nil {
|
||||
whereClause += " AND n.categoria_id = $" + strconv.Itoa(argNum)
|
||||
whereClause += " AND n.categoria_id = $" + strconv.Itoa(argNum)
|
||||
catID, err := strconv.ParseInt(categoriaID, 10, 64)
|
||||
if err == nil {
|
||||
args = append(args, catID)
|
||||
argNum++
|
||||
}
|
||||
}
|
||||
|
||||
if paisID != "" {
|
||||
if pID, err := strconv.ParseInt(paisID, 10, 64); err == nil {
|
||||
whereClause += " AND n.pais_id = $" + strconv.Itoa(argNum)
|
||||
whereClause += " AND n.pais_id = $" + strconv.Itoa(argNum)
|
||||
pID, err := strconv.ParseInt(paisID, 10, 64)
|
||||
if err == nil {
|
||||
args = append(args, pID)
|
||||
argNum++
|
||||
}
|
||||
}
|
||||
|
||||
sqlQuery := `
|
||||
SELECT n.id, n.titulo, COALESCE(n.resumen, ''), n.contenido, n.url, n.fecha, n.imagen_url,
|
||||
n.categoria_id, n.pais_id, n.fuente_nombre, n.lang,
|
||||
t.titulo_trad,
|
||||
t.resumen_trad,
|
||||
t.lang_to as lang_trad
|
||||
FROM noticias n
|
||||
LEFT JOIN traducciones t ON t.noticia_id = n.id AND t.lang_to = $1
|
||||
` + whereClause + `
|
||||
ORDER BY n.fecha DESC
|
||||
LIMIT $` + strconv.Itoa(argNum) + ` OFFSET $` + strconv.Itoa(argNum+1)
|
||||
|
||||
args = append(args, perPage, offset)
|
||||
|
||||
sqlQuery := `
|
||||
SELECT n.id, n.titulo, n.resumen, n.contenido, n.url, n.imagen,
|
||||
n.feed_id, n.lang, n.categoria_id, n.pais_id, n.created_at, n.updated_at,
|
||||
COALESCE(t.titulo_trad, '') as titulo_trad,
|
||||
COALESCE(t.resumen_trad, '') as resumen_trad,
|
||||
t.lang_to as lang_trad,
|
||||
f.nombre as fuente_nombre
|
||||
FROM noticias n
|
||||
LEFT JOIN traducciones t ON t.noticia_id = n.id AND t.lang_to = $` + strconv.Itoa(argNum) + `
|
||||
LEFT JOIN feeds f ON f.id = n.feed_id
|
||||
` + whereClause + `
|
||||
ORDER BY n.created_at DESC
|
||||
LIMIT $` + strconv.Itoa(argNum+1) + ` OFFSET $` + strconv.Itoa(argNum+2)
|
||||
|
||||
rows, err := db.GetPool().Query(ctx, sqlQuery, args...)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, models.ErrorResponse{Error: "Search failed", Message: err.Error()})
|
||||
|
|
@ -191,37 +111,30 @@ func SearchNews(c *gin.Context) {
|
|||
var newsList []models.NewsWithTranslations
|
||||
for rows.Next() {
|
||||
var n models.NewsWithTranslations
|
||||
var imagenURL, fuenteNombre *string
|
||||
var categoriaIDp, paisIDp *int64
|
||||
var lang string
|
||||
var imagen *string
|
||||
|
||||
err := rows.Scan(
|
||||
&n.ID, &n.Titulo, &n.Resumen, &n.Contenido, &n.URL, &n.Fecha, &imagenURL,
|
||||
&categoriaIDp, &paisIDp, &fuenteNombre, &lang,
|
||||
&n.ID, &n.Title, &n.Summary, &n.Content, &n.URL, &imagen,
|
||||
&n.FeedID, &n.Lang, &n.CategoryID, &n.CountryID, &n.CreatedAt, &n.UpdatedAt,
|
||||
&n.TitleTranslated, &n.SummaryTranslated, &n.LangTranslated,
|
||||
)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if imagenURL != nil {
|
||||
n.ImagenURL = imagenURL
|
||||
if imagen != nil {
|
||||
n.ImageURL = imagen
|
||||
}
|
||||
if fuenteNombre != nil {
|
||||
n.FuenteNombre = *fuenteNombre
|
||||
}
|
||||
n.CategoryID = categoriaIDp
|
||||
n.CountryID = paisIDp
|
||||
n.Lang = lang
|
||||
newsList = append(newsList, n)
|
||||
}
|
||||
|
||||
// Get total count
|
||||
countArgs := args[:len(args)-2]
|
||||
|
||||
// Remove LIMIT/OFFSET from args for count
|
||||
var total int
|
||||
err = db.GetPool().QueryRow(ctx, `
|
||||
SELECT COUNT(*) FROM noticias n
|
||||
LEFT JOIN traducciones t ON t.noticia_id = n.id AND t.lang_to = $1
|
||||
LEFT JOIN traducciones t ON t.noticia_id = n.id AND t.lang_to = $`+strconv.Itoa(argNum)+`
|
||||
`+whereClause, countArgs...).Scan(&total)
|
||||
if err != nil {
|
||||
total = len(newsList)
|
||||
|
|
@ -229,42 +142,21 @@ func SearchNews(c *gin.Context) {
|
|||
|
||||
totalPages := (total + perPage - 1) / perPage
|
||||
|
||||
response := gin.H{
|
||||
"news": newsList,
|
||||
"total": total,
|
||||
"page": page,
|
||||
"per_page": perPage,
|
||||
"total_pages": totalPages,
|
||||
response := models.NewsListResponse{
|
||||
News: newsList,
|
||||
Total: total,
|
||||
Page: page,
|
||||
PerPage: perPage,
|
||||
TotalPages: totalPages,
|
||||
}
|
||||
|
||||
// Cache the response
|
||||
if data, err := json.Marshal(response); err == nil {
|
||||
cache.Set(ctx, cacheKey, string(data), cache.TTLShort)
|
||||
}
|
||||
|
||||
c.JSON(http.StatusOK, response)
|
||||
c.JSON(http.StatusOK, response)
|
||||
}
|
||||
|
||||
// GetStats returns global statistics
|
||||
// @Summary Get statistics
|
||||
// @Description Returns global statistics about news, feeds, users, and translations
|
||||
// @Tags stats
|
||||
// @Produce json
|
||||
// @Success 200 {object} models.Stats
|
||||
// @Failure 500 {object} models.ErrorResponse
|
||||
// @Router /stats [get]
|
||||
func GetStats(c *gin.Context) {
|
||||
ctx := c.Request.Context()
|
||||
cacheKey := cache.StatsKey()
|
||||
|
||||
if cached, err := cache.Get(ctx, cacheKey); err == nil && cached != "" {
|
||||
c.Data(http.StatusOK, "application/json", []byte(cached))
|
||||
return
|
||||
}
|
||||
|
||||
var stats models.Stats
|
||||
|
||||
err := db.GetPool().QueryRow(ctx, `
|
||||
err := db.GetPool().QueryRow(c.Request.Context(), `
|
||||
SELECT
|
||||
(SELECT COUNT(*) FROM noticias) as total_news,
|
||||
(SELECT COUNT(*) FROM feeds WHERE activo = true) as total_feeds,
|
||||
|
|
@ -283,7 +175,7 @@ func GetStats(c *gin.Context) {
|
|||
return
|
||||
}
|
||||
|
||||
rows, err := db.GetPool().Query(ctx, `
|
||||
rows, err := db.GetPool().Query(c.Request.Context(), `
|
||||
SELECT c.id, c.nombre, COUNT(n.id) as count
|
||||
FROM categorias c
|
||||
LEFT JOIN noticias n ON n.categoria_id = c.id
|
||||
|
|
@ -295,12 +187,12 @@ func GetStats(c *gin.Context) {
|
|||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
var cs models.CategoryStat
|
||||
rows.Scan(&cs.CategoriaID, &cs.CategoriaName, &cs.Count)
|
||||
rows.Scan(&cs.CategoryID, &cs.CategoryName, &cs.Count)
|
||||
stats.TopCategories = append(stats.TopCategories, cs)
|
||||
}
|
||||
}
|
||||
|
||||
rows, err = db.GetPool().Query(ctx, `
|
||||
rows, err = db.GetPool().Query(c.Request.Context(), `
|
||||
SELECT p.id, p.nombre, p.flag_emoji, COUNT(n.id) as count
|
||||
FROM paises p
|
||||
LEFT JOIN noticias n ON n.pais_id = p.id
|
||||
|
|
@ -312,28 +204,16 @@ func GetStats(c *gin.Context) {
|
|||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
var cs models.CountryStat
|
||||
rows.Scan(&cs.PaisID, &cs.PaisName, &cs.FlagEmoji, &cs.Count)
|
||||
rows.Scan(&cs.CountryID, &cs.CountryName, &cs.FlagEmoji, &cs.Count)
|
||||
stats.TopCountries = append(stats.TopCountries, cs)
|
||||
}
|
||||
}
|
||||
|
||||
if data, err := json.Marshal(stats); err == nil {
|
||||
cache.Set(ctx, cacheKey, string(data), cache.TTLMedium)
|
||||
}
|
||||
|
||||
c.JSON(http.StatusOK, stats)
|
||||
}
|
||||
|
||||
func GetCategories(c *gin.Context) {
|
||||
ctx := c.Request.Context()
|
||||
cacheKey := cache.CategoriesKey()
|
||||
|
||||
if cached, err := cache.Get(ctx, cacheKey); err == nil && cached != "" {
|
||||
c.Data(http.StatusOK, "application/json", []byte(cached))
|
||||
return
|
||||
}
|
||||
|
||||
rows, err := db.GetPool().Query(ctx, `
|
||||
rows, err := db.GetPool().Query(c.Request.Context(), `
|
||||
SELECT id, nombre FROM categorias ORDER BY nombre`)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, models.ErrorResponse{Error: "Failed to get categories", Message: err.Error()})
|
||||
|
|
@ -341,30 +221,23 @@ func GetCategories(c *gin.Context) {
|
|||
}
|
||||
defer rows.Close()
|
||||
|
||||
var categories []models.Category
|
||||
for rows.Next() {
|
||||
var cat models.Category
|
||||
rows.Scan(&cat.ID, &cat.Nombre)
|
||||
categories = append(categories, cat)
|
||||
type Category struct {
|
||||
ID int64 `json:"id"`
|
||||
Nombre string `json:"nombre"`
|
||||
}
|
||||
|
||||
if data, err := json.Marshal(categories); err == nil {
|
||||
cache.Set(ctx, cacheKey, string(data), cache.TTLLong)
|
||||
var categories []Category
|
||||
for rows.Next() {
|
||||
var cat Category
|
||||
rows.Scan(&cat.ID, &cat.Nombre)
|
||||
categories = append(categories, cat)
|
||||
}
|
||||
|
||||
c.JSON(http.StatusOK, categories)
|
||||
}
|
||||
|
||||
func GetCountries(c *gin.Context) {
|
||||
ctx := c.Request.Context()
|
||||
cacheKey := cache.CountriesKey()
|
||||
|
||||
if cached, err := cache.Get(ctx, cacheKey); err == nil && cached != "" {
|
||||
c.Data(http.StatusOK, "application/json", []byte(cached))
|
||||
return
|
||||
}
|
||||
|
||||
rows, err := db.GetPool().Query(ctx, `
|
||||
rows, err := db.GetPool().Query(c.Request.Context(), `
|
||||
SELECT p.id, p.nombre, c.nombre as continente
|
||||
FROM paises p
|
||||
LEFT JOIN continentes c ON c.id = p.continente_id
|
||||
|
|
@ -375,16 +248,18 @@ func GetCountries(c *gin.Context) {
|
|||
}
|
||||
defer rows.Close()
|
||||
|
||||
var countries []models.Country
|
||||
type Country struct {
|
||||
ID int64 `json:"id"`
|
||||
Nombre string `json:"nombre"`
|
||||
Continente string `json:"continente"`
|
||||
}
|
||||
|
||||
var countries []Country
|
||||
for rows.Next() {
|
||||
var country models.Country
|
||||
var country Country
|
||||
rows.Scan(&country.ID, &country.Nombre, &country.Continente)
|
||||
countries = append(countries, country)
|
||||
}
|
||||
|
||||
if data, err := json.Marshal(countries); err == nil {
|
||||
cache.Set(ctx, cacheKey, string(data), cache.TTLLong)
|
||||
}
|
||||
|
||||
c.JSON(http.StatusOK, countries)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,35 +0,0 @@
|
|||
package handlers
|
||||
|
||||
import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestValidatePageParamsSearch(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
page int
|
||||
perPage int
|
||||
defaultPerPage int
|
||||
maxPerPage int
|
||||
expectedPage int
|
||||
expectedPerPage int
|
||||
}{
|
||||
{"normal", 1, 30, 30, 100, 1, 30},
|
||||
{"page zero", 0, 30, 30, 100, 1, 30},
|
||||
{"per_page too large", 1, 200, 30, 100, 1, 100},
|
||||
{"page negative", -5, 30, 30, 100, 1, 30},
|
||||
{"per_page zero", 1, 0, 30, 100, 1, 30},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
page, perPage := validatePageParams(tt.page, tt.perPage, tt.defaultPerPage, tt.maxPerPage)
|
||||
if page != tt.expectedPage {
|
||||
t.Errorf("page: expected %d, got %d", tt.expectedPage, page)
|
||||
}
|
||||
if perPage != tt.expectedPerPage {
|
||||
t.Errorf("perPage: expected %d, got %d", tt.expectedPerPage, perPage)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
@ -1,14 +0,0 @@
|
|||
package handlers
|
||||
|
||||
func validatePageParams(page, perPage, defaultPerPage, maxPerPage int) (int, int) {
|
||||
if page < 1 {
|
||||
page = 1
|
||||
}
|
||||
if perPage < 1 {
|
||||
perPage = defaultPerPage
|
||||
}
|
||||
if perPage > maxPerPage {
|
||||
perPage = maxPerPage
|
||||
}
|
||||
return page, perPage
|
||||
}
|
||||
|
|
@ -1,344 +0,0 @@
|
|||
package handlers
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"log"
|
||||
"net/http"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/gorilla/websocket"
|
||||
"github.com/rss2/backend/internal/db"
|
||||
"github.com/rss2/backend/internal/models"
|
||||
)
|
||||
|
||||
var upgrader = websocket.Upgrader{
|
||||
CheckOrigin: func(r *http.Request) bool { return true },
|
||||
}
|
||||
|
||||
type WSWorker struct {
|
||||
conn *websocket.Conn
|
||||
workerID int
|
||||
capabilities string
|
||||
workerName string
|
||||
lastHeartbeat time.Time
|
||||
mu sync.Mutex
|
||||
}
|
||||
|
||||
var (
|
||||
workers = make(map[int]*WSWorker)
|
||||
workersByAPI = make(map[string]*WSWorker)
|
||||
workersMu sync.RWMutex
|
||||
)
|
||||
|
||||
func HandleWorkerWS(c *gin.Context) {
|
||||
apiKey := c.Query("api_key")
|
||||
if apiKey == "" {
|
||||
apiKey = c.GetHeader("X-API-Key")
|
||||
}
|
||||
|
||||
if apiKey == "" {
|
||||
c.JSON(http.StatusUnauthorized, gin.H{"error": "API key required"})
|
||||
return
|
||||
}
|
||||
|
||||
ctx := c.Request.Context()
|
||||
|
||||
var workerID int
|
||||
var capabilities string
|
||||
err := db.GetPool().QueryRow(ctx, `
|
||||
SELECT id, capabilities FROM remote_workers
|
||||
WHERE api_key = $1 AND status != 'disabled'
|
||||
`, apiKey).Scan(&workerID, &capabilities)
|
||||
|
||||
if err != nil {
|
||||
c.JSON(http.StatusUnauthorized, gin.H{"error": "Invalid API key"})
|
||||
return
|
||||
}
|
||||
|
||||
conn, err := upgrader.Upgrade(c.Writer, c.Request, nil)
|
||||
if err != nil {
|
||||
log.Printf("WebSocket upgrade error: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
wsWorker := &WSWorker{
|
||||
conn: conn,
|
||||
workerID: workerID,
|
||||
capabilities: capabilities,
|
||||
lastHeartbeat: time.Now(),
|
||||
}
|
||||
|
||||
workersMu.Lock()
|
||||
workers[workerID] = wsWorker
|
||||
workersByAPI[apiKey] = wsWorker
|
||||
workersMu.Unlock()
|
||||
|
||||
db.GetPool().Exec(ctx, `
|
||||
UPDATE remote_workers SET status = 'online', last_seen = NOW() WHERE id = $1
|
||||
`, workerID)
|
||||
|
||||
go heartbeatLoop(wsWorker)
|
||||
go writeLoop(wsWorker)
|
||||
readLoop(wsWorker, apiKey)
|
||||
}
|
||||
|
||||
func readLoop(wsWorker *WSWorker, apiKey string) {
|
||||
defer func() {
|
||||
cleanupWorker(wsWorker, apiKey)
|
||||
}()
|
||||
|
||||
for {
|
||||
var msg models.WSClientMessage
|
||||
err := wsWorker.conn.ReadJSON(&msg)
|
||||
if err != nil {
|
||||
log.Printf("Worker %d read error: %v", wsWorker.workerID, err)
|
||||
break
|
||||
}
|
||||
|
||||
switch msg.Type {
|
||||
case "register":
|
||||
wsWorker.mu.Lock()
|
||||
if msg.WorkerName != "" {
|
||||
wsWorker.workerName = msg.WorkerName
|
||||
}
|
||||
if msg.Capabilities != "" {
|
||||
wsWorker.capabilities = msg.Capabilities
|
||||
}
|
||||
wsWorker.mu.Unlock()
|
||||
|
||||
sendWS(wsWorker, models.WSServerMessage{Type: "ack", Job: nil})
|
||||
|
||||
case "heartbeat":
|
||||
wsWorker.mu.Lock()
|
||||
wsWorker.lastHeartbeat = time.Now()
|
||||
wsWorker.mu.Unlock()
|
||||
sendWS(wsWorker, models.WSServerMessage{Type: "ack", Job: nil})
|
||||
|
||||
case "result":
|
||||
if msg.Result != nil {
|
||||
handleTranslationResult(wsWorker.workerID, msg.Result)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func writeLoop(wsWorker *WSWorker) {
|
||||
ticker := time.NewTicker(30 * time.Second)
|
||||
defer ticker.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ticker.C:
|
||||
wsWorker.mu.Lock()
|
||||
err := wsWorker.conn.WriteMessage(websocket.PingMessage, nil)
|
||||
wsWorker.mu.Unlock()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func heartbeatLoop(wsWorker *WSWorker) {
|
||||
ticker := time.NewTicker(10 * time.Second)
|
||||
defer ticker.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ticker.C:
|
||||
wsWorker.mu.Lock()
|
||||
elapsed := time.Since(wsWorker.lastHeartbeat)
|
||||
wsWorker.mu.Unlock()
|
||||
|
||||
if elapsed > 60*time.Second {
|
||||
log.Printf("Worker %d heartbeat timeout", wsWorker.workerID)
|
||||
wsWorker.conn.Close()
|
||||
return
|
||||
}
|
||||
|
||||
sendWS(wsWorker, models.WSServerMessage{Type: "ping", Job: nil})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func sendWS(wsWorker *WSWorker, msg models.WSServerMessage) {
|
||||
data, _ := json.Marshal(msg)
|
||||
wsWorker.mu.Lock()
|
||||
wsWorker.conn.WriteMessage(websocket.TextMessage, data)
|
||||
wsWorker.mu.Unlock()
|
||||
}
|
||||
|
||||
func cleanupWorker(wsWorker *WSWorker, apiKey string) {
|
||||
workersMu.Lock()
|
||||
delete(workers, wsWorker.workerID)
|
||||
delete(workersByAPI, apiKey)
|
||||
workersMu.Unlock()
|
||||
|
||||
ctx := context.Background()
|
||||
db.GetPool().Exec(ctx, `
|
||||
UPDATE remote_workers SET status = 'offline', last_seen = NOW() WHERE id = $1
|
||||
`, wsWorker.workerID)
|
||||
|
||||
wsWorker.conn.Close()
|
||||
|
||||
db.GetPool().Exec(ctx, `
|
||||
UPDATE traducciones
|
||||
SET status = 'pending', worker_id = NULL, assigned_at = NULL
|
||||
WHERE worker_id = $1 AND status = 'assigned'
|
||||
`, wsWorker.workerID)
|
||||
|
||||
log.Printf("Worker %d disconnected", wsWorker.workerID)
|
||||
}
|
||||
|
||||
func handleTranslationResult(workerID int, result *models.TranslationResult) {
|
||||
ctx := context.Background()
|
||||
|
||||
if result.Error != "" {
|
||||
db.GetPool().Exec(ctx, `
|
||||
UPDATE traducciones
|
||||
SET status = 'error', worker_id = NULL, assigned_at = NULL
|
||||
WHERE id = $1
|
||||
`, result.JobID)
|
||||
log.Printf("Job %d failed: %s", result.JobID, result.Error)
|
||||
return
|
||||
}
|
||||
|
||||
_, err := db.GetPool().Exec(ctx, `
|
||||
UPDATE traducciones
|
||||
SET titulo_trad = $1, resumen_trad = $2, status = 'done',
|
||||
worker_id = $3, assigned_at = NULL
|
||||
WHERE id = $4
|
||||
`, result.TitleTr, result.SummaryTr, workerID, result.JobID)
|
||||
|
||||
if err != nil {
|
||||
log.Printf("Error updating translation result: %v", err)
|
||||
}
|
||||
|
||||
db.GetPool().Exec(ctx, `
|
||||
UPDATE remote_workers SET last_seen = NOW() WHERE id = $1
|
||||
`, workerID)
|
||||
|
||||
log.Printf("Job %d completed by worker %d", result.JobID, workerID)
|
||||
}
|
||||
|
||||
func AssignJobToWorker(workerID int) *models.TranslationJob {
|
||||
ctx := context.Background()
|
||||
|
||||
workersMu.RLock()
|
||||
wsWorker, exists := workers[workerID]
|
||||
workersMu.RUnlock()
|
||||
|
||||
if !exists {
|
||||
return nil
|
||||
}
|
||||
|
||||
wsWorker.mu.Lock()
|
||||
workerCapabilities := wsWorker.capabilities
|
||||
wsWorker.mu.Unlock()
|
||||
|
||||
_ = workerCapabilities
|
||||
|
||||
var job models.TranslationJob
|
||||
err := db.GetPool().QueryRow(ctx, `
|
||||
SELECT t.id, t.noticia_id, t.lang_from, t.lang_to, n.titulo, n.resumen
|
||||
FROM traducciones t
|
||||
JOIN noticias n ON n.id = t.noticia_id
|
||||
WHERE t.status = 'pending'
|
||||
AND (t.worker_id IS NULL OR t.status = 'assigned' AND t.assigned_at < NOW() - INTERVAL '5 minutes')
|
||||
AND t.lang_to = 'es'
|
||||
AND (t.titulo_trad IS NULL OR t.resumen_trad IS NULL)
|
||||
ORDER BY n.fecha DESC
|
||||
LIMIT 1
|
||||
FOR UPDATE SKIP LOCKED
|
||||
`).Scan(&job.ID, &job.NewsID, &job.LangFrom, &job.LangTo, &job.Title, &job.Summary)
|
||||
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
db.GetPool().Exec(ctx, `
|
||||
UPDATE traducciones
|
||||
SET status = 'assigned', worker_id = $1, assigned_at = NOW()
|
||||
WHERE id = $2
|
||||
`, workerID, job.ID)
|
||||
|
||||
sendWS(wsWorker, models.WSServerMessage{
|
||||
Type: "job",
|
||||
Job: &job,
|
||||
})
|
||||
|
||||
return &job
|
||||
}
|
||||
|
||||
func StartJobAssigner() {
|
||||
go func() {
|
||||
ticker := time.NewTicker(5 * time.Second)
|
||||
defer ticker.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ticker.C:
|
||||
workersMu.RLock()
|
||||
for workerID := range workers {
|
||||
workersMu.RUnlock()
|
||||
AssignJobToWorker(workerID)
|
||||
workersMu.RLock()
|
||||
}
|
||||
workersMu.RUnlock()
|
||||
|
||||
ctx := context.Background()
|
||||
db.GetPool().Exec(ctx, `
|
||||
UPDATE traducciones
|
||||
SET status = 'pending', worker_id = NULL, assigned_at = NULL
|
||||
WHERE status = 'assigned'
|
||||
AND assigned_at < NOW() - INTERVAL '10 minutes'
|
||||
`)
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
func GetModelDownloadURL(c *gin.Context) {
|
||||
apiKey := c.Query("api_key")
|
||||
if apiKey == "" {
|
||||
apiKey = c.GetHeader("X-API-Key")
|
||||
}
|
||||
|
||||
if apiKey == "" {
|
||||
c.JSON(http.StatusUnauthorized, gin.H{"error": "API key required"})
|
||||
return
|
||||
}
|
||||
|
||||
ctx := c.Request.Context()
|
||||
|
||||
var workerID int
|
||||
err := db.GetPool().QueryRow(ctx, `
|
||||
SELECT id FROM remote_workers WHERE api_key = $1
|
||||
`, apiKey).Scan(&workerID)
|
||||
|
||||
if err != nil {
|
||||
c.JSON(http.StatusUnauthorized, gin.H{"error": "Invalid API key"})
|
||||
return
|
||||
}
|
||||
|
||||
var modelURL, modelPath string
|
||||
db.GetPool().QueryRow(ctx, `
|
||||
SELECT value FROM config WHERE key = 'remote_worker_model_url'
|
||||
`).Scan(&modelURL)
|
||||
|
||||
db.GetPool().QueryRow(ctx, `
|
||||
SELECT value FROM config WHERE key = 'remote_worker_model_path'
|
||||
`).Scan(&modelPath)
|
||||
|
||||
if modelPath == "" {
|
||||
modelPath = "/app/models/nllb-ct2"
|
||||
}
|
||||
|
||||
c.JSON(http.StatusOK, gin.H{
|
||||
"model_download_url": modelURL,
|
||||
"model_path": modelPath,
|
||||
})
|
||||
}
|
||||
|
|
@ -1,60 +0,0 @@
|
|||
package logger
|
||||
|
||||
import (
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/rs/zerolog"
|
||||
"github.com/rs/zerolog/log"
|
||||
)
|
||||
|
||||
var Logger zerolog.Logger
|
||||
|
||||
func Init(serviceName, level string) {
|
||||
zerolog.TimeFieldFormat = zerolog.TimeFormatUnixMs
|
||||
zerolog.CallerMarshalFunc = func(pc uintptr, file string, line int) string {
|
||||
short := file
|
||||
for i := len(file) - 1; i > 0; i-- {
|
||||
if file[i] == '/' {
|
||||
short = file[i+1:]
|
||||
break
|
||||
}
|
||||
}
|
||||
return short + ":" + string(rune(line))
|
||||
}
|
||||
|
||||
output := zerolog.ConsoleWriter{Out: os.Stdout, TimeFormat: time.RFC3339, NoColor: false}
|
||||
logger := zerolog.New(output).With().Timestamp().Caller().Str("service", serviceName).Logger()
|
||||
|
||||
switch level {
|
||||
case "debug":
|
||||
logger = logger.Level(zerolog.DebugLevel)
|
||||
case "info":
|
||||
logger = logger.Level(zerolog.InfoLevel)
|
||||
case "warn":
|
||||
logger = logger.Level(zerolog.WarnLevel)
|
||||
case "error":
|
||||
logger = logger.Level(zerolog.ErrorLevel)
|
||||
default:
|
||||
logger = logger.Level(zerolog.InfoLevel)
|
||||
}
|
||||
|
||||
Logger = logger
|
||||
log.Logger = logger
|
||||
}
|
||||
|
||||
func GetLogger() zerolog.Logger {
|
||||
return Logger
|
||||
}
|
||||
|
||||
func WithRequestID(requestID string) zerolog.Logger {
|
||||
return Logger.With().Str("request_id", requestID).Logger()
|
||||
}
|
||||
|
||||
func WithFields(fields map[string]interface{}) zerolog.Logger {
|
||||
ctx := Logger.With()
|
||||
for k, v := range fields {
|
||||
ctx = ctx.Interface(k, v)
|
||||
}
|
||||
return ctx.Logger()
|
||||
}
|
||||
|
|
@ -1,19 +1,27 @@
|
|||
package middleware
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/rss2/backend/internal/auth"
|
||||
"github.com/rss2/backend/internal/config"
|
||||
"github.com/rss2/backend/internal/logger"
|
||||
"golang.org/x/time/rate"
|
||||
"github.com/golang-jwt/jwt/v5"
|
||||
)
|
||||
|
||||
var jwtSecret []byte
|
||||
|
||||
func SetJWTSecret(secret string) {
|
||||
jwtSecret = []byte(secret)
|
||||
}
|
||||
|
||||
type Claims struct {
|
||||
UserID int64 `json:"user_id"`
|
||||
Email string `json:"email"`
|
||||
Username string `json:"username"`
|
||||
IsAdmin bool `json:"is_admin"`
|
||||
jwt.RegisteredClaims
|
||||
}
|
||||
|
||||
func AuthRequired() gin.HandlerFunc {
|
||||
return func(c *gin.Context) {
|
||||
authHeader := c.GetHeader("Authorization")
|
||||
|
|
@ -30,8 +38,12 @@ func AuthRequired() gin.HandlerFunc {
|
|||
return
|
||||
}
|
||||
|
||||
claims, err := auth.ValidateToken(tokenString)
|
||||
if err != nil {
|
||||
claims := &Claims{}
|
||||
token, err := jwt.ParseWithClaims(tokenString, claims, func(token *jwt.Token) (interface{}, error) {
|
||||
return jwtSecret, nil
|
||||
})
|
||||
|
||||
if err != nil || !token.Valid {
|
||||
c.JSON(http.StatusUnauthorized, gin.H{"error": "Invalid token"})
|
||||
c.Abort()
|
||||
return
|
||||
|
|
@ -51,7 +63,7 @@ func AdminRequired() gin.HandlerFunc {
|
|||
return
|
||||
}
|
||||
|
||||
claims := userVal.(*auth.Claims)
|
||||
claims := userVal.(*Claims)
|
||||
if !claims.IsAdmin {
|
||||
c.JSON(http.StatusForbidden, gin.H{"error": "Admin access required"})
|
||||
c.Abort()
|
||||
|
|
@ -63,26 +75,9 @@ func AdminRequired() gin.HandlerFunc {
|
|||
}
|
||||
|
||||
func CORSMiddleware() gin.HandlerFunc {
|
||||
cfg := config.Load()
|
||||
allowedOrigins := strings.Split(cfg.AllowedOrigins, ",")
|
||||
|
||||
return func(c *gin.Context) {
|
||||
origin := c.Request.Header.Get("Origin")
|
||||
|
||||
allowed := false
|
||||
for _, o := range allowedOrigins {
|
||||
o = strings.TrimSpace(o)
|
||||
if o == origin {
|
||||
allowed = true
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if allowed {
|
||||
c.Writer.Header().Set("Access-Control-Allow-Origin", origin)
|
||||
c.Writer.Header().Set("Access-Control-Allow-Credentials", "true")
|
||||
}
|
||||
|
||||
c.Writer.Header().Set("Access-Control-Allow-Origin", "*")
|
||||
c.Writer.Header().Set("Access-Control-Allow-Credentials", "true")
|
||||
c.Writer.Header().Set("Access-Control-Allow-Headers", "Content-Type, Content-Length, Accept-Encoding, X-CSRF-Token, Authorization, accept, origin, Cache-Control, X-Requested-With")
|
||||
c.Writer.Header().Set("Access-Control-Allow-Methods", "POST, OPTIONS, GET, PUT, DELETE, PATCH")
|
||||
|
||||
|
|
@ -97,101 +92,17 @@ func CORSMiddleware() gin.HandlerFunc {
|
|||
|
||||
func LoggerMiddleware() gin.HandlerFunc {
|
||||
return func(c *gin.Context) {
|
||||
start := time.Now()
|
||||
path := c.Request.URL.Path
|
||||
raw := c.Request.URL.RawQuery
|
||||
requestID := c.GetHeader("X-Request-ID")
|
||||
if requestID == "" {
|
||||
requestID = c.GetString("request_id")
|
||||
}
|
||||
|
||||
c.Next()
|
||||
|
||||
latency := time.Since(start)
|
||||
status := c.Writer.Status()
|
||||
clientIP := c.ClientIP()
|
||||
method := c.Request.Method
|
||||
|
||||
if raw != "" {
|
||||
path = path + "?" + raw
|
||||
}
|
||||
|
||||
log := logger.GetLogger().With().
|
||||
Str("method", method).
|
||||
Str("path", path).
|
||||
Int("status", status).
|
||||
Str("client_ip", clientIP).
|
||||
Dur("latency", latency).
|
||||
Logger()
|
||||
|
||||
if requestID != "" {
|
||||
log = log.With().Str("request_id", requestID).Logger()
|
||||
}
|
||||
|
||||
if status >= 500 {
|
||||
log.Error().Msg("Server error")
|
||||
} else if status >= 400 {
|
||||
log.Warn().Msg("Client error")
|
||||
} else {
|
||||
log.Info().Msg("Request completed")
|
||||
if status >= 400 {
|
||||
// Log error responses
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type clientLimiter struct {
|
||||
limiter *rate.Limiter
|
||||
lastSeen time.Time
|
||||
}
|
||||
|
||||
var (
|
||||
clientLimiters = make(map[string]*clientLimiter)
|
||||
limitersMu sync.Mutex
|
||||
)
|
||||
|
||||
func RateLimitMiddleware(requestsPerMinute int) gin.HandlerFunc {
|
||||
limit := rate.Limit(requestsPerMinute) / 60
|
||||
burst := requestsPerMinute
|
||||
|
||||
return func(c *gin.Context) {
|
||||
ip := c.ClientIP()
|
||||
|
||||
limitersMu.Lock()
|
||||
cl, exists := clientLimiters[ip]
|
||||
if !exists {
|
||||
cl = &clientLimiter{limiter: rate.NewLimiter(limit, burst)}
|
||||
clientLimiters[ip] = cl
|
||||
}
|
||||
cl.lastSeen = time.Now()
|
||||
limiter := cl.limiter
|
||||
limitersMu.Unlock()
|
||||
|
||||
if !limiter.Allow() {
|
||||
c.JSON(http.StatusTooManyRequests, gin.H{"error": "Rate limit exceeded"})
|
||||
c.Abort()
|
||||
return
|
||||
}
|
||||
|
||||
go func() {
|
||||
time.Sleep(10 * time.Minute)
|
||||
limitersMu.Lock()
|
||||
if cl, ok := clientLimiters[ip]; ok && time.Since(cl.lastSeen) > 10*time.Minute {
|
||||
delete(clientLimiters, ip)
|
||||
}
|
||||
limitersMu.Unlock()
|
||||
}()
|
||||
|
||||
c.Next()
|
||||
}
|
||||
}
|
||||
|
||||
func RequestIDMiddleware() gin.HandlerFunc {
|
||||
return func(c *gin.Context) {
|
||||
requestID := c.GetHeader("X-Request-ID")
|
||||
if requestID == "" {
|
||||
requestID = fmt.Sprintf("%d", time.Now().UnixNano())
|
||||
}
|
||||
c.Set("request_id", requestID)
|
||||
c.Header("X-Request-ID", requestID)
|
||||
c.Next()
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,141 +0,0 @@
|
|||
package middleware
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/rss2/backend/internal/auth"
|
||||
)
|
||||
|
||||
func init() {
|
||||
gin.SetMode(gin.TestMode)
|
||||
auth.SetJWTSecret("test-secret-key-12345")
|
||||
}
|
||||
|
||||
func TestAuthRequiredNoHeader(t *testing.T) {
|
||||
router := gin.New()
|
||||
router.Use(AuthRequired())
|
||||
router.GET("/protected", func(c *gin.Context) {
|
||||
c.JSON(200, gin.H{"status": "ok"})
|
||||
})
|
||||
|
||||
req, _ := http.NewRequest("GET", "/protected", nil)
|
||||
w := httptest.NewRecorder()
|
||||
router.ServeHTTP(w, req)
|
||||
|
||||
if w.Code != http.StatusUnauthorized {
|
||||
t.Errorf("expected status 401, got %d", w.Code)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAuthRequiredInvalidToken(t *testing.T) {
|
||||
router := gin.New()
|
||||
router.Use(AuthRequired())
|
||||
router.GET("/protected", func(c *gin.Context) {
|
||||
c.JSON(200, gin.H{"status": "ok"})
|
||||
})
|
||||
|
||||
req, _ := http.NewRequest("GET", "/protected", nil)
|
||||
req.Header.Set("Authorization", "Bearer invalid-token")
|
||||
w := httptest.NewRecorder()
|
||||
router.ServeHTTP(w, req)
|
||||
|
||||
if w.Code != http.StatusUnauthorized {
|
||||
t.Errorf("expected status 401, got %d", w.Code)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAuthRequiredValidToken(t *testing.T) {
|
||||
router := gin.New()
|
||||
router.Use(AuthRequired())
|
||||
router.GET("/protected", func(c *gin.Context) {
|
||||
userVal, _ := c.Get("user")
|
||||
claims := userVal.(*auth.Claims)
|
||||
c.JSON(200, gin.H{"user_id": claims.UserID})
|
||||
})
|
||||
|
||||
token, _ := auth.GenerateToken(42, "test@test.com", "testuser", false)
|
||||
|
||||
req, _ := http.NewRequest("GET", "/protected", nil)
|
||||
req.Header.Set("Authorization", "Bearer "+token)
|
||||
w := httptest.NewRecorder()
|
||||
router.ServeHTTP(w, req)
|
||||
|
||||
if w.Code != http.StatusOK {
|
||||
t.Errorf("expected status 200, got %d", w.Code)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAdminRequiredNotAdmin(t *testing.T) {
|
||||
router := gin.New()
|
||||
router.Use(AuthRequired())
|
||||
router.Use(AdminRequired())
|
||||
router.GET("/admin", func(c *gin.Context) {
|
||||
c.JSON(200, gin.H{"status": "ok"})
|
||||
})
|
||||
|
||||
token, _ := auth.GenerateToken(1, "test@test.com", "testuser", false)
|
||||
|
||||
req, _ := http.NewRequest("GET", "/admin", nil)
|
||||
req.Header.Set("Authorization", "Bearer "+token)
|
||||
w := httptest.NewRecorder()
|
||||
router.ServeHTTP(w, req)
|
||||
|
||||
if w.Code != http.StatusForbidden {
|
||||
t.Errorf("expected status 403, got %d", w.Code)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAdminRequiredIsAdmin(t *testing.T) {
|
||||
router := gin.New()
|
||||
router.Use(AuthRequired())
|
||||
router.Use(AdminRequired())
|
||||
router.GET("/admin", func(c *gin.Context) {
|
||||
c.JSON(200, gin.H{"status": "ok"})
|
||||
})
|
||||
|
||||
token, _ := auth.GenerateToken(1, "test@test.com", "adminuser", true)
|
||||
|
||||
req, _ := http.NewRequest("GET", "/admin", nil)
|
||||
req.Header.Set("Authorization", "Bearer "+token)
|
||||
w := httptest.NewRecorder()
|
||||
router.ServeHTTP(w, req)
|
||||
|
||||
if w.Code != http.StatusOK {
|
||||
t.Errorf("expected status 200, got %d", w.Code)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCORSMiddleware(t *testing.T) {
|
||||
router := gin.New()
|
||||
router.Use(CORSMiddleware())
|
||||
router.GET("/test", func(c *gin.Context) {
|
||||
c.JSON(200, gin.H{"status": "ok"})
|
||||
})
|
||||
|
||||
req, _ := http.NewRequest("GET", "/test", nil)
|
||||
w := httptest.NewRecorder()
|
||||
router.ServeHTTP(w, req)
|
||||
|
||||
if w.Header().Get("Access-Control-Allow-Origin") == "" {
|
||||
t.Error("expected CORS header to be set")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCORSPreflight(t *testing.T) {
|
||||
router := gin.New()
|
||||
router.Use(CORSMiddleware())
|
||||
router.GET("/test", func(c *gin.Context) {
|
||||
c.JSON(200, gin.H{"status": "ok"})
|
||||
})
|
||||
|
||||
req, _ := http.NewRequest("OPTIONS", "/test", nil)
|
||||
w := httptest.NewRecorder()
|
||||
router.ServeHTTP(w, req)
|
||||
|
||||
if w.Code != http.StatusNoContent {
|
||||
t.Errorf("expected status 204 for preflight, got %d", w.Code)
|
||||
}
|
||||
}
|
||||
|
|
@ -6,12 +6,12 @@ import (
|
|||
|
||||
type News struct {
|
||||
ID int64 `json:"id"`
|
||||
Titulo string `json:"titulo"`
|
||||
Resumen string `json:"resumen"`
|
||||
Contenido string `json:"contenido"`
|
||||
Title string `json:"title"`
|
||||
Summary string `json:"summary"`
|
||||
Content string `json:"content"`
|
||||
URL string `json:"url"`
|
||||
ImagenURL *string `json:"imagen_url"`
|
||||
Fecha *time.Time `json:"fecha"`
|
||||
ImageURL *string `json:"image_url"`
|
||||
PublishedAt *time.Time `json:"published_at"`
|
||||
Lang string `json:"lang"`
|
||||
FeedID int64 `json:"feed_id"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
|
|
@ -20,73 +20,35 @@ type News struct {
|
|||
|
||||
type NewsWithTranslations struct {
|
||||
ID int64 `json:"id"`
|
||||
Titulo string `json:"titulo"`
|
||||
Resumen string `json:"resumen"`
|
||||
Contenido string `json:"contenido"`
|
||||
Title string `json:"title"`
|
||||
Summary string `json:"summary"`
|
||||
Content string `json:"content"`
|
||||
URL string `json:"url"`
|
||||
ImagenURL *string `json:"imagen_url"`
|
||||
Fecha *string `json:"fecha"`
|
||||
ImageURL *string `json:"image_url"`
|
||||
PublishedAt *string `json:"published_at"`
|
||||
Lang string `json:"lang"`
|
||||
FeedID int64 `json:"feed_id"`
|
||||
CategoryID *int64 `json:"categoria_id"`
|
||||
CountryID *int64 `json:"pais_id"`
|
||||
FuenteNombre string `json:"fuente_nombre"`
|
||||
CategoryID *int64 `json:"category_id"`
|
||||
CountryID *int64 `json:"country_id"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
TitleTranslated *string `json:"title_translated"`
|
||||
SummaryTranslated *string `json:"summary_translated"`
|
||||
ContentTranslated *string `json:"content_translated"`
|
||||
LangTranslated *string `json:"lang_translated"`
|
||||
Entities []Entity `json:"entities,omitempty"`
|
||||
}
|
||||
|
||||
type Entity struct {
|
||||
Valor string `json:"valor"`
|
||||
Tipo string `json:"tipo"`
|
||||
Count int `json:"count"`
|
||||
WikiSummary *string `json:"wiki_summary"`
|
||||
WikiURL *string `json:"wiki_url"`
|
||||
ImagePath *string `json:"image_path"`
|
||||
}
|
||||
|
||||
type EntityListResponse struct {
|
||||
Entities []Entity `json:"entities"`
|
||||
Total int `json:"total"`
|
||||
Page int `json:"page"`
|
||||
PerPage int `json:"per_page"`
|
||||
TotalPages int `json:"total_pages"`
|
||||
}
|
||||
|
||||
type MentionPoint struct {
|
||||
Fecha string `json:"fecha"`
|
||||
Count int `json:"count"`
|
||||
}
|
||||
|
||||
type MentionSeries struct {
|
||||
Valor string `json:"valor"`
|
||||
Tipo string `json:"tipo"`
|
||||
Count int `json:"count"`
|
||||
Data []MentionPoint `json:"data"`
|
||||
}
|
||||
|
||||
type MentionsResponse struct {
|
||||
Days int `json:"days"`
|
||||
Series []MentionSeries `json:"series"`
|
||||
}
|
||||
|
||||
type Feed struct {
|
||||
ID int64 `json:"id"`
|
||||
Nombre string `json:"nombre"`
|
||||
Title string `json:"title"`
|
||||
URL string `json:"url"`
|
||||
SiteURL *string `json:"site_url"`
|
||||
Descripcion *string `json:"descripcion"`
|
||||
ImagenURL *string `json:"imagen_url"`
|
||||
Idioma *string `json:"idioma"`
|
||||
CategoriaID *int64 `json:"categoria_id"`
|
||||
PaisID *int64 `json:"pais_id"`
|
||||
Activo bool `json:"activo"`
|
||||
Fallos *int64 `json:"fallos"`
|
||||
LastError *string `json:"last_error"`
|
||||
Description *string `json:"description"`
|
||||
ImageURL *string `json:"image_url"`
|
||||
Language *string `json:"language"`
|
||||
CategoryID *int64 `json:"category_id"`
|
||||
CountryID *int64 `json:"country_id"`
|
||||
Active bool `json:"active"`
|
||||
LastFetched *time.Time `json:"last_fetched"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
|
|
@ -94,27 +56,27 @@ type Feed struct {
|
|||
|
||||
type Category struct {
|
||||
ID int64 `json:"id"`
|
||||
Nombre string `json:"nombre"`
|
||||
Name string `json:"name"`
|
||||
Color string `json:"color"`
|
||||
Icon string `json:"icon"`
|
||||
ParentID *int64 `json:"parent_id"`
|
||||
}
|
||||
|
||||
type Country struct {
|
||||
ID int64 `json:"id"`
|
||||
Nombre string `json:"nombre"`
|
||||
Codigo string `json:"codigo"`
|
||||
Continente string `json:"continente"`
|
||||
FlagEmoji string `json:"flag_emoji"`
|
||||
ID int64 `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Code string `json:"code"`
|
||||
Continent string `json:"continent"`
|
||||
FlagEmoji string `json:"flag_emoji"`
|
||||
}
|
||||
|
||||
type Translation struct {
|
||||
ID int64 `json:"id"`
|
||||
NoticiaID int64 `json:"noticia_id"`
|
||||
NewsID int64 `json:"news_id"`
|
||||
LangFrom string `json:"lang_from"`
|
||||
LangTo string `json:"lang_to"`
|
||||
Titulo string `json:"titulo"`
|
||||
Resumen string `json:"resumen"`
|
||||
Title string `json:"title"`
|
||||
Summary string `json:"summary"`
|
||||
Status string `json:"status"`
|
||||
Error *string `json:"error"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
|
|
@ -127,7 +89,6 @@ type User struct {
|
|||
Username string `json:"username"`
|
||||
PasswordHash string `json:"-"`
|
||||
IsAdmin bool `json:"is_admin"`
|
||||
Role string `json:"role"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
}
|
||||
|
|
@ -136,8 +97,8 @@ type SearchHistory struct {
|
|||
ID int64 `json:"id"`
|
||||
UserID int64 `json:"user_id"`
|
||||
Query string `json:"query"`
|
||||
CategoriaID *int64 `json:"categoria_id"`
|
||||
PaisID *int64 `json:"pais_id"`
|
||||
CategoryID *int64 `json:"category_id"`
|
||||
CountryID *int64 `json:"country_id"`
|
||||
ResultsCount int `json:"results_count"`
|
||||
SearchedAt time.Time `json:"searched_at"`
|
||||
}
|
||||
|
|
@ -171,14 +132,14 @@ type Stats struct {
|
|||
}
|
||||
|
||||
type CategoryStat struct {
|
||||
CategoriaID int64 `json:"categoria_id"`
|
||||
CategoriaName string `json:"categoria_nombre"`
|
||||
Count int64 `json:"count"`
|
||||
CategoryID int64 `json:"category_id"`
|
||||
CategoryName string `json:"category_name"`
|
||||
Count int64 `json:"count"`
|
||||
}
|
||||
|
||||
type CountryStat struct {
|
||||
PaisID int64 `json:"pais_id"`
|
||||
PaisName string `json:"pais_nombre"`
|
||||
CountryID int64 `json:"country_id"`
|
||||
CountryName string `json:"country_name"`
|
||||
FlagEmoji string `json:"flag_emoji"`
|
||||
Count int64 `json:"count"`
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,41 +0,0 @@
|
|||
package models
|
||||
|
||||
import "time"
|
||||
|
||||
type RemoteWorker struct {
|
||||
ID int `json:"id"`
|
||||
Name string `json:"name"`
|
||||
APIKey string `json:"api_key,omitempty"`
|
||||
Capabilities string `json:"capabilities"`
|
||||
Status string `json:"status"`
|
||||
LastSeen *time.Time `json:"last_seen,omitempty"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
}
|
||||
|
||||
type TranslationJob struct {
|
||||
ID int64 `json:"id"`
|
||||
NewsID int64 `json:"noticia_id"`
|
||||
LangFrom string `json:"lang_from"`
|
||||
LangTo string `json:"lang_to"`
|
||||
Title string `json:"title"`
|
||||
Summary string `json:"summary"`
|
||||
}
|
||||
|
||||
type TranslationResult struct {
|
||||
JobID int64 `json:"job_id"`
|
||||
TitleTr string `json:"title_trad"`
|
||||
SummaryTr string `json:"resumen_trad"`
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
type WSClientMessage struct {
|
||||
Type string `json:"type"`
|
||||
Capabilities string `json:"capabilities,omitempty"`
|
||||
WorkerName string `json:"worker_name,omitempty"`
|
||||
Result *TranslationResult `json:"result,omitempty"`
|
||||
}
|
||||
|
||||
type WSServerMessage struct {
|
||||
Type string `json:"type"`
|
||||
Job *TranslationJob `json:"job,omitempty"`
|
||||
}
|
||||
|
|
@ -8,9 +8,10 @@ import (
|
|||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
"github.com/rss2/backend/internal/db"
|
||||
)
|
||||
|
||||
var pool *pgxpool.Pool
|
||||
|
||||
type Config struct {
|
||||
Host string
|
||||
Port int
|
||||
|
|
@ -30,23 +31,39 @@ func LoadDBConfig() *Config {
|
|||
}
|
||||
|
||||
func Connect(cfg *Config) error {
|
||||
// Just verify the main pool is accessible
|
||||
if db.GetPool() == nil {
|
||||
return fmt.Errorf("database pool not initialized")
|
||||
dsn := fmt.Sprintf("postgres://%s:%s@%s:%d/%s?sslmode=disable",
|
||||
cfg.User, cfg.Password, cfg.Host, cfg.Port, cfg.DBName)
|
||||
|
||||
poolConfig, err := pgxpool.ParseConfig(dsn)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to parse config: %w", err)
|
||||
}
|
||||
return db.HealthCheck(context.Background())
|
||||
|
||||
poolConfig.MaxConns = 25
|
||||
poolConfig.MinConns = 5
|
||||
poolConfig.MaxConnLifetime = time.Hour
|
||||
poolConfig.MaxConnIdleTime = 30 * time.Minute
|
||||
|
||||
pool, err = pgxpool.NewWithConfig(context.Background(), poolConfig)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create pool: %w", err)
|
||||
}
|
||||
|
||||
if err = pool.Ping(context.Background()); err != nil {
|
||||
return fmt.Errorf("failed to ping database: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func GetPool() *pgxpool.Pool {
|
||||
return db.GetPool()
|
||||
return pool
|
||||
}
|
||||
|
||||
func Close() {
|
||||
// No-op: main db package manages the pool
|
||||
}
|
||||
|
||||
func HealthCheck(ctx context.Context) error {
|
||||
return db.HealthCheck(ctx)
|
||||
if pool != nil {
|
||||
pool.Close()
|
||||
}
|
||||
}
|
||||
|
||||
func getEnv(key, defaultValue string) string {
|
||||
|
|
|
|||
16
backup.sh
16
backup.sh
|
|
@ -1,16 +0,0 @@
|
|||
#!/bin/bash
|
||||
# Backup diario de PostgreSQL rss2
|
||||
set -e
|
||||
|
||||
BACKUP_DIR="/home/x/rss2/backups"
|
||||
TIMESTAMP=$(date +%Y%m%d_%H%M%S)
|
||||
OUT="$BACKUP_DIR/rss-${TIMESTAMP}.sql.gz"
|
||||
LOG="$BACKUP_DIR/backup.log"
|
||||
|
||||
mkdir -p "$BACKUP_DIR"
|
||||
docker exec rss2_db pg_dump -U rss -d rss --no-owner --no-privileges | gzip > "$OUT"
|
||||
|
||||
# Retener solo los ultimos 30 dias (borrado seguro de ficheros concretos, nunca rm -rf)
|
||||
find "$BACKUP_DIR" -maxdepth 1 -name 'rss-*.sql.gz' -mtime +30 -delete
|
||||
|
||||
echo "$(date '+%F %T') Backup OK: $OUT ($(du -h "$OUT" | cut -f1))" >> "$LOG"
|
||||
448
bugs.md
448
bugs.md
|
|
@ -1,448 +0,0 @@
|
|||
# Análisis de Errores e Inconsistencias - RSS2
|
||||
|
||||
Fecha de análisis: 2026-04-01
|
||||
|
||||
---
|
||||
|
||||
## Resumen Ejecutivo
|
||||
|
||||
Se han identificado **13 problemas** clasificados en:
|
||||
- 3 Errores Críticos
|
||||
- 7 Inconsistencias
|
||||
- 3 Warnings de Arquitectura
|
||||
|
||||
---
|
||||
|
||||
## 1. Errores Críticos
|
||||
|
||||
### 1.1 Rate Limiting No Implementado
|
||||
|
||||
**Ubicación:** `backend/internal/middleware/auth.go:104-108`
|
||||
|
||||
**Descripción:**
|
||||
La función existe pero no hace nada:
|
||||
```go
|
||||
func RateLimitMiddleware(requestsPerMinute int) gin.HandlerFunc {
|
||||
return func(c *gin.Context) {
|
||||
c.Next()
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
**Problema:**
|
||||
No implementa ningún tipo de limitación de rate. Vulnerable a ataques de fuerza bruta y DoS.
|
||||
|
||||
**Impacto:** Alto - Seguridad comprometida.
|
||||
|
||||
---
|
||||
|
||||
### 1.2 CORS Demasiado Permisivo
|
||||
|
||||
**Ubicación:** `backend/internal/middleware/auth.go:77-90`
|
||||
|
||||
**Descripción:**
|
||||
```go
|
||||
c.Writer.Header().Set("Access-Control-Allow-Origin", "*")
|
||||
```
|
||||
|
||||
**Problema:**
|
||||
Permite cualquier origen, lo cual es inseguro para APIs de producción. Permite ataques CSRF y expone datos a orígenes maliciosos.
|
||||
|
||||
**Impacto:** Alto - Seguridad comprometida.
|
||||
|
||||
---
|
||||
|
||||
### 1.3 Secret Key Por Defecto Hardcodeada
|
||||
|
||||
**Ubicación:** `backend/internal/config/config.go:32`
|
||||
|
||||
**Descripción:**
|
||||
```go
|
||||
SecretKey: getEnv("SECRET_KEY", "change-this-secret-key"),
|
||||
```
|
||||
|
||||
**Problema:**
|
||||
Si no se proporciona la variable de entorno, la aplicación usa una clave por defecto conocida. Cualquier despliegue sin configurar esta variable es vulnerable.
|
||||
|
||||
**Impacto:** Crítico - Compromiso total de autenticación.
|
||||
|
||||
---
|
||||
|
||||
## 2. Inconsistencias
|
||||
|
||||
### 2.1 Mapeo de Campos Inconsistente (Snake Case vs Camel Case)
|
||||
|
||||
**Descripción:**
|
||||
La aplicación usa convenciones de nomenclatura diferentes en distintas capas:
|
||||
|
||||
| Capa | Estructura | Ejemplo |
|
||||
|------|------------|---------|
|
||||
| Models | Snake case | `Title`, `Summary`, `ImageURL` |
|
||||
| Handlers | Pascal case | `Titulo`, `Resumen`, `ImagenURL` |
|
||||
| Frontend TS | camelCase | `titulo`, `resumen`, `imagen_url` |
|
||||
|
||||
**Archivos afectados:**
|
||||
- `backend/internal/models/models.go:7-19`
|
||||
- `backend/internal/handlers/news.go:14-28`
|
||||
- `frontend/src/services/api.ts:20-34`
|
||||
|
||||
**Problema:**
|
||||
Dificulta el mantenimiento y genera potenciales errores de serialización.
|
||||
|
||||
---
|
||||
|
||||
### 2.2 Idioma de Traducción Hardcodeado
|
||||
|
||||
**Descripción:**
|
||||
El idioma destino de traducción (`es`) está hardcodeado en múltiples lugares:
|
||||
|
||||
**news.go:67:**
|
||||
```go
|
||||
where += " AND t.status = 'done' AND t.titulo_trad IS NOT NULL"
|
||||
```
|
||||
|
||||
**news.go:96:**
|
||||
```go
|
||||
LEFT JOIN traducciones t ON t.noticia_id = n.id AND t.lang_to = 'es'
|
||||
```
|
||||
|
||||
**search.go:34-37:**
|
||||
```go
|
||||
if lang == "" {
|
||||
lang = "es"
|
||||
}
|
||||
```
|
||||
|
||||
**Problema:**
|
||||
No es configurable. Si se quiere traducir a otros idiomas, requiere cambios en múltiples archivos.
|
||||
|
||||
---
|
||||
|
||||
### 2.3 Columna `role` en Users Sin映射 en Modelo
|
||||
|
||||
**Descripción:**
|
||||
Se añade una columna `role` a la tabla users:
|
||||
|
||||
**cmd/server/main.go:41-48:**
|
||||
```go
|
||||
_, err = db.GetPool().Exec(ctx, `
|
||||
ALTER TABLE users ADD COLUMN IF NOT EXISTS role VARCHAR(20) DEFAULT 'user'
|
||||
`)
|
||||
```
|
||||
|
||||
**Pero el modelo User no tiene este campo:**
|
||||
|
||||
**models/models.go:86-94:**
|
||||
```go
|
||||
type User struct {
|
||||
ID int64 `json:"id"`
|
||||
Email string `json:"email"`
|
||||
Username string `json:"username"`
|
||||
PasswordHash string `json:"-"`
|
||||
IsAdmin bool `json:"is_admin"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
}
|
||||
```
|
||||
|
||||
**Problema:**
|
||||
La columna `role` nunca se usa realmente. Hay dos campos (`role` y `is_admin`) que pueden causar confusión.
|
||||
|
||||
---
|
||||
|
||||
### 2.4 Feature No Implementada en Discovery
|
||||
|
||||
**Ubicación:** `backend/cmd/discovery/main.go:129-133`
|
||||
|
||||
**Descripción:**
|
||||
```go
|
||||
func findFeedLinksInHTML(baseURL string) ([]string, error) {
|
||||
// Simple feed link finder - returns empty for now
|
||||
// In production, use goquery to parse HTML and find RSS/Atom links
|
||||
return []string{}, nil
|
||||
}
|
||||
```
|
||||
|
||||
**Problema:**
|
||||
El worker de discovery no puede encontrar feeds RSS embebidos en páginas HTML. Solo detecta feeds directos.
|
||||
|
||||
**Impacto:** Medio - Reduce la efectividad del descubrimiento de feeds.
|
||||
|
||||
---
|
||||
|
||||
### 2.5 LoggerMiddleware No Funcional
|
||||
|
||||
**Ubicación:** `backend/internal/middleware/auth.go:93-101`
|
||||
|
||||
**Descripción:**
|
||||
```go
|
||||
func LoggerMiddleware() gin.HandlerFunc {
|
||||
return func(c *gin.Context) {
|
||||
c.Next()
|
||||
|
||||
status := c.Writer.Status()
|
||||
if status >= 400 {
|
||||
// Log error responses - pero no hace nada!
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
**Problema:**
|
||||
El logging de errores está commented out. No se registra nada.
|
||||
|
||||
---
|
||||
|
||||
### 2.6 Columnas de BD Diferentes a Modelos
|
||||
|
||||
**Descripción:**
|
||||
El handler `search.go` referencia columnas que no existen en los modelos:
|
||||
|
||||
**search.go:91-92:**
|
||||
```go
|
||||
SELECT n.id, n.titulo, n.resumen, n.contenido, n.url, n.imagen,
|
||||
n.feed_id, n.lang, n.categoria_id, n.pais_id, n.created_at, n.updated_at,
|
||||
```
|
||||
|
||||
**Pero en models.go:7-19:**
|
||||
```go
|
||||
type News struct {
|
||||
ID int64 `json:"id"`
|
||||
Title string `json:"title"`
|
||||
Summary string `json:"summary"`
|
||||
Content string `json:"content"`
|
||||
URL string `json:"url"`
|
||||
ImageURL *string `json:"image_url"`
|
||||
PublishedAt *time.Time `json:"published_at"`
|
||||
Lang string `json:"lang"`
|
||||
FeedID int64 `json:"feed_id"`
|
||||
...
|
||||
}
|
||||
```
|
||||
|
||||
**Problema:**
|
||||
- Modelo usa `Title`, BD usa `titulo`
|
||||
- Modelo usa `Summary`, BD usa `resumen`
|
||||
- Modelo usa `ImageURL`, BD usa `imagen`
|
||||
- Modelo usa `PublishedAt`, BD usa `fecha`
|
||||
|
||||
Esto causa errores de serialización potenciales.
|
||||
|
||||
---
|
||||
|
||||
### 2.7 Inicialización de Base de Datos Duplicada
|
||||
|
||||
**Descripción:**
|
||||
Existen dos formas de conectar a la base de datos:
|
||||
|
||||
1. `db.Connect()` en `backend/internal/db/postgres.go:13`
|
||||
2. `workers.Connect()` en `backend/internal/workers/db.go`
|
||||
|
||||
**Problema:**
|
||||
Códigos duplicados que mantienen conexiones separadas. Puede causar problemas de pool de conexiones.
|
||||
|
||||
---
|
||||
|
||||
## 3. Warnings de Arquitectura
|
||||
|
||||
### 3.1 Creación de Tablas en Runtime
|
||||
|
||||
**Ubicación:** `backend/cmd/server/main.go:20-77`
|
||||
|
||||
**Descripción:**
|
||||
En lugar de usar migraciones formales, el servidor crea tablas al iniciar:
|
||||
|
||||
```go
|
||||
func initDB() {
|
||||
_, err := db.GetPool().Exec(ctx, `
|
||||
CREATE TABLE IF NOT EXISTS entity_aliases (...)
|
||||
`)
|
||||
// más tablas...
|
||||
}
|
||||
```
|
||||
|
||||
**Problema:**
|
||||
- No hay control de versiones
|
||||
- No hay rollback
|
||||
- Difícil de mantener en producción
|
||||
|
||||
**Recomendación:** Usar golang-migrate o similar.
|
||||
|
||||
---
|
||||
|
||||
### 3.2 No Hay Tests Unitarios
|
||||
|
||||
**Descripción:**
|
||||
No se encontraron archivos `*_test.go` en el código backend.
|
||||
|
||||
**Problema:**
|
||||
Sin tests, cualquier cambio puede romper funcionalidad existente sin detección.
|
||||
|
||||
---
|
||||
|
||||
### 3.3 Pool de Conexiones Sin Validación Periódica
|
||||
|
||||
**Ubicación:** `backend/internal/db/postgres.go`
|
||||
|
||||
**Descripción:**
|
||||
El pool se crea pero no hay health check periódico.
|
||||
|
||||
**Problema:**
|
||||
Las conexiones zombie pueden acumularse sin detección.
|
||||
|
||||
---
|
||||
|
||||
## 4. Recomendaciones de Corrección
|
||||
|
||||
### Alta Prioridad
|
||||
|
||||
1. **Unificar JWT Secret:** Usar un único package para manejar el secret
|
||||
2. **Implementar Rate Limiting:** Usar una librería como `golang.org/x/time/rate`
|
||||
3. **Configurar CORS correctamente:** Usar variable de entorno para orígenes permitidos
|
||||
4. **Forzar Secret Key:** Hacer requerida la variable `SECRET_KEY`
|
||||
|
||||
### Media Prioridad
|
||||
|
||||
5. **Traducir idioma configurable:** Pasar idioma como parámetro o configuración
|
||||
6. **Usar migraciones:** Implementar golang-migrate
|
||||
7. **Añadir tests:** Crear tests unitarios para handlers críticos
|
||||
8. **Corregir mapeo de campos:** Estandarizar snake_case en toda la aplicación
|
||||
|
||||
### Baja Prioridad
|
||||
|
||||
9. **Implementar LoggerMiddleware:** Añadir logging real
|
||||
10. **Implementar findFeedLinksInHTML:** Usar goquery para parsing HTML
|
||||
11. **Consolidar DB:** Unificar inicialización de base de datos
|
||||
|
||||
---
|
||||
|
||||
## 5. Estadísticas
|
||||
|
||||
| Tipo | Cantidad |
|
||||
|------|----------|
|
||||
| Errores Críticos | 3 |
|
||||
| Inconsistencias | 7 |
|
||||
| Warnings | 3 |
|
||||
| **Total** | **13** |
|
||||
|
||||
---
|
||||
|
||||
## 6. Problemas Adicionales Detectados
|
||||
|
||||
### 6.1 Mapeo de Campos Inconsistente (Frontend)
|
||||
|
||||
**Descripción:** El frontend usa `snake_case` pero los modelos Go usan `PascalCase`.
|
||||
|
||||
**Archivos:**
|
||||
- `frontend/src/services/api.ts` - usa `titulo`, `resumen`
|
||||
- `backend/internal/models/models.go` - usa `Title`, `Summary`
|
||||
|
||||
---
|
||||
|
||||
### 6.2 Structs Duplicados para Mismo Recurso
|
||||
|
||||
**Descripción:** `handlers/news.go` define `NewsResponse` diferente a `models/models.go:News`.
|
||||
|
||||
---
|
||||
|
||||
### 6.3 Query Params Inconsistentes
|
||||
|
||||
**Descripción:** Algunos handlers usan `category_id` (inglés) y otros `categoria_id` (español).
|
||||
|
||||
---
|
||||
|
||||
### 6.4 Duplicación Config DB
|
||||
|
||||
**Descripción:** API usa `DATABASE_URL`, workers usan `DB_HOST`, `DB_PORT`, etc.
|
||||
|
||||
---
|
||||
|
||||
### 6.5 Sin Health Checks en Workers
|
||||
|
||||
**Descripción:** Workers no implementan endpoint `/health`.
|
||||
|
||||
---
|
||||
|
||||
### 6.6 Manejo de Errores Inconsistente
|
||||
|
||||
**Descripción:** Mezcla de `gin.H{"error": ...}` y `models.ErrorResponse`.
|
||||
|
||||
---
|
||||
|
||||
## 7. Errores y Problemas Críticos Detectados
|
||||
|
||||
### 7.1 ~~Errores Ignorados Silenciosamente~~ **(RESUELTO)**
|
||||
|
||||
**Descripción:** Múltiples lugares donde errores eran ignorados sin logging.
|
||||
|
||||
**Ejemplos corregidos:**
|
||||
- `admin.go:488-492` - Ahora verifica errores de `db.GetPool().Exec()`
|
||||
- `admin.go:522` - Verifica error al actualizar estado
|
||||
|
||||
**Impacto:** Fallos silenciosos que pasan desapercibidos.
|
||||
|
||||
---
|
||||
|
||||
### 7.2 ~~Rutas Hardcodeadas~~ **(RESUELTO)**
|
||||
|
||||
**Descripción:** Rutas de archivos hardcodeadas en el código.
|
||||
|
||||
**Correcciones:**
|
||||
- `admin.go:469,474,509` - Ahora usa `config.Load().DockerComposeDir`
|
||||
- `server/main.go:113` - Ahora usa `cfg.WikiImagesPath`
|
||||
- Añadidos nuevos campos en config: `DockerComposeDir`, `WikiImagesPath`
|
||||
|
||||
---
|
||||
|
||||
## 7.3 Secret Key Por Defecto
|
||||
|
||||
**Descripción:** `config.go:32` usa clave por defecto known.
|
||||
|
||||
```go
|
||||
SecretKey: getEnv("SECRET_KEY", "change-this-secret-key")
|
||||
```
|
||||
|
||||
**Impacto:** Medio - Si no se configura, usar clave insegura.
|
||||
|
||||
---
|
||||
|
||||
### 7.4 Rate Limiting No Implementado
|
||||
|
||||
**Descripción:** `middleware/auth.go:104-108` tiene función vacía.
|
||||
|
||||
**Impacto:** Alto - Vulnerable a ataques de fuerza bruta.
|
||||
|
||||
---
|
||||
|
||||
### 7.5 CORS Demasiado Permisivo
|
||||
|
||||
**Descripción:** `middleware/auth.go:79` permite cualquier origen.
|
||||
|
||||
**Impacto:** Alto - Expuesto a ataques CSRF.
|
||||
|
||||
---
|
||||
|
||||
## 8. Recomendaciones de Corrección (Actualizado)
|
||||
|
||||
### Alta Prioridad (CRÍTICO)
|
||||
|
||||
1. ~~Unificar JWT Secret~~ ✅
|
||||
2. Implementar Rate Limiting
|
||||
3. ~~**Configurar CORS correctamente:** Usar variable de entorno~~ ✅
|
||||
4. ~~**Corregir manejo de errores:** No ignorar silenciosamente~~ ✅
|
||||
5. ~~**Eliminar rutas hardcodeadas**~~ ✅
|
||||
|
||||
### Media Prioridad
|
||||
|
||||
7. Idioma traducciones configurable
|
||||
8. Usar migraciones
|
||||
9. Añadir tests
|
||||
10. Corregir mapeo de campos
|
||||
|
||||
### Baja Prioridad
|
||||
|
||||
11. Implementar LoggerMiddleware
|
||||
12. Implementar findFeedLinksInHTML
|
||||
13. Consolidar DB
|
||||
14. Estandarizar respuestas API
|
||||
15. Añadir health checks
|
||||
|
|
@ -99,16 +99,8 @@
|
|||
"Pakistán",
|
||||
"Tolo News",
|
||||
"Gladbach",
|
||||
"Alger",
|
||||
"Comisión",
|
||||
"Dirección",
|
||||
"Más",
|
||||
"Regístrese",
|
||||
"Consejo",
|
||||
"Fondo",
|
||||
"PVT LTD",
|
||||
"Beneficios",
|
||||
"Ingrese"
|
||||
"Alger"
|
||||
|
||||
],
|
||||
"synonyms": {
|
||||
"Alassane Ouattara": [
|
||||
18
data/feeds.csv
Normal file
18
data/feeds.csv
Normal file
|
|
@ -0,0 +1,18 @@
|
|||
id,nombre,descripcion,url,categoria_id,categoria,pais_id,pais,idioma,activo,fallos
|
||||
19,8am Daily – Dari,روزنامه ۸صبح افغانستان به زبان دری.,https://8am.af/feed,7,Internacional,1,Afganistán,fa,False,30
|
||||
20,8am Daily – Pashto,د ۸صبح ورځپاڼې پښتو خپرونه.,https://8am.af/ps/feed,7,Internacional,1,Afganistán,ps,False,30
|
||||
1,Afghanistan News.Net – Noticias,Cobertura continua de noticias generales sobre Afganistán y su entorno regional.,https://feeds.afghanistannews.net/rss/6e1d5c8e1f98f17c,7,Internacional,1,Afganistán,en,True,0
|
||||
36,Arezo TV – Dari,آرزو تلویزیون – خبر و گزارش به دری.,https://arezo.tv/fa/feed,7,Internacional,1,Afganistán,fa,False,30
|
||||
37,Arezo TV – Pashto,د آرزو تلویزیون پښتو خبرونه.,https://arezo.tv/ps/feed,7,Internacional,1,Afganistán,ps,False,30
|
||||
4,Ariana News – Dari,خبرها و تحلیلها از افغانستان به زبان دری.,https://ariananews.af/feed,7,Internacional,1,Afganistán,fa,True,0
|
||||
28,Avapress – Dari,خبرگزاری صدای افغان (آوا) به زبان دری.,https://avapress.com/fa/rss,7,Internacional,1,Afganistán,fa,False,30
|
||||
29,Avapress – Pashto,د افغان غږ خبري آژانس په پښتو ژبه.,https://avapress.com/ps/rss,7,Internacional,1,Afganistán,ps,False,30
|
||||
23,Bakhtar News Agency – Dari,آژانس خبری باختر به زبان دری.,https://bakhtarnews.af/fa/feed,7,Internacional,1,Afganistán,fa,False,5
|
||||
24,Bakhtar News Agency – Pashto,د باختر خبري اژانس پښتو پاڼه.,https://bakhtarnews.af/ps/feed,7,Internacional,1,Afganistán,ps,False,5
|
||||
38,Barya News – Dari,باریانیوز – رسانه خبری افغانستان.,https://barya.news/feed,7,Internacional,1,Afganistán,fa,False,30
|
||||
39,Barya News – Pashto,باریانیوز پښتو خپرونې.,https://barya.news/ps/feed,7,Internacional,1,Afganistán,ps,False,30
|
||||
47,Chaprast News – Dari,چپرست نیوز – خبرهای افغانستان.,https://chaprast.com/feed,7,Internacional,1,Afganistán,fa,False,30
|
||||
30,Ensaf News – Dari,رسانه تحلیلی به زبان دری.,https://www.ensafnews.com/fa/feed,7,Internacional,1,Afganistán,fa,False,30
|
||||
27,Hamshahri Afghanistan – Dari,نسخه افغانستان همشهری به زبان دری.,https://hamshahri.af/feed,7,Internacional,1,Afganistán,fa,False,30
|
||||
44,Jomhor News – Dari,جمهور نیوز – خبرگزاری مستقل دری.,https://jomhornews.com/fa/rss,7,Internacional,1,Afganistán,fa,False,30
|
||||
45,Jomhor News – Pashto,جمهور نیوز پښتو.,https://jomhornews.com/ps/rss,7,Internacional,1,Afganistán,ps,False,30
|
||||
|
|
|
@ -1,47 +0,0 @@
|
|||
#!/bin/bash
|
||||
# Script para despliegue limpio de RSS2
|
||||
|
||||
echo "=== RSS2 Clean Deployment Script ==="
|
||||
echo ""
|
||||
|
||||
# Detener contenedores
|
||||
echo "1. Deteniendo contenedores..."
|
||||
docker compose down -v 2>/dev/null
|
||||
|
||||
# Eliminar volúmenes de datos (si hay permisos)
|
||||
echo "2. Eliminando volúmenes de datos..."
|
||||
docker volume rm rss2_db 2>/dev/null || true
|
||||
docker volume rm rss2_redis 2>/dev/null || true
|
||||
|
||||
# Si los volúmenes Docker tienen problemas, intentar con rm
|
||||
echo " Intentando limpiar /data/..."
|
||||
sudo rm -rf /datos/rss2/data/pgdata 2>/dev/null || true
|
||||
sudo rm -rf /datos/rss2/data/redis-data 2>/dev/null || true
|
||||
|
||||
# Iniciar base de datos
|
||||
echo "3. Iniciando base de datos..."
|
||||
docker compose up -d db
|
||||
|
||||
# Esperar a que esté lista
|
||||
echo "4. Esperando a que la base de datos esté lista..."
|
||||
sleep 10
|
||||
|
||||
# Verificar estado
|
||||
if docker compose ps db | grep -q "healthy"; then
|
||||
echo " ✓ Base de datos iniciada correctamente"
|
||||
|
||||
# Ejecutar script de schema
|
||||
echo "5. Ejecutando script de inicialización..."
|
||||
docker compose exec -T db psql -U rss -d rss -f /docker-entrypoint-initdb.d/00-complete-schema.sql 2>&1 | tail -5
|
||||
|
||||
# Iniciar demás servicios
|
||||
echo "6. Iniciando servicios..."
|
||||
docker compose up -d redis backend-go rss2_frontend nginx rss-ingestor-go
|
||||
|
||||
echo ""
|
||||
echo "=== Despliegue completado ==="
|
||||
echo "Accede a: http://localhost:8001"
|
||||
else
|
||||
echo " ✗ Error: La base de datos no está healthy"
|
||||
docker compose logs db
|
||||
fi
|
||||
74
deploy/debian/build.sh
Executable file
74
deploy/debian/build.sh
Executable file
|
|
@ -0,0 +1,74 @@
|
|||
#!/usr/bin/env bash
|
||||
# =============================================================================
|
||||
# RSS2 - Recompila binarios y frontend (sin reinstalar el sistema)
|
||||
# Usar despues de actualizar el codigo: bash build.sh
|
||||
# =============================================================================
|
||||
set -euo pipefail
|
||||
|
||||
RSS2_HOME="/opt/rss2"
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
|
||||
|
||||
export PATH=$PATH:/usr/local/go/bin
|
||||
|
||||
GREEN='\033[0;32m'; YELLOW='\033[1;33m'; NC='\033[0m'
|
||||
info() { echo -e "${GREEN}[BUILD]${NC} $*"; }
|
||||
warn() { echo -e "${YELLOW}[WARN]${NC} $*"; }
|
||||
|
||||
# --- Go Backend + Workers ---
|
||||
if [[ -d "$REPO_ROOT/backend" ]]; then
|
||||
info "Compilando backend Go..."
|
||||
(cd "$REPO_ROOT/backend" && \
|
||||
CGO_ENABLED=0 GOOS=linux go build -buildvcs=false -o "$RSS2_HOME/bin/server" ./cmd/server)
|
||||
info " [OK] server"
|
||||
|
||||
for cmd in scraper discovery wiki_worker topics related; do
|
||||
[[ -d "$REPO_ROOT/backend/cmd/$cmd" ]] || continue
|
||||
(cd "$REPO_ROOT/backend" && \
|
||||
CGO_ENABLED=0 GOOS=linux go build -buildvcs=false -o "$RSS2_HOME/bin/$cmd" "./cmd/$cmd")
|
||||
info " [OK] $cmd"
|
||||
done
|
||||
# qdrant worker: nombre del binario debe coincidir con el service
|
||||
[[ -d "$REPO_ROOT/backend/cmd/qdrant" ]] && \
|
||||
(cd "$REPO_ROOT/backend" && \
|
||||
CGO_ENABLED=0 GOOS=linux go build -buildvcs=false -o "$RSS2_HOME/bin/qdrant_worker" "./cmd/qdrant")
|
||||
info " [OK] qdrant_worker"
|
||||
fi
|
||||
|
||||
# --- Ingestor Go ---
|
||||
if [[ -d "$REPO_ROOT/rss-ingestor-go" ]]; then
|
||||
info "Compilando ingestor Go..."
|
||||
(cd "$REPO_ROOT/rss-ingestor-go" && \
|
||||
GOTOOLCHAIN=local CGO_ENABLED=0 GOOS=linux go build -buildvcs=false -o "$RSS2_HOME/bin/ingestor" .)
|
||||
info " [OK] ingestor"
|
||||
fi
|
||||
|
||||
# --- Frontend React ---
|
||||
if [[ -d "$REPO_ROOT/frontend" ]]; then
|
||||
info "Compilando frontend React..."
|
||||
(cd "$REPO_ROOT/frontend" && \
|
||||
npm install --silent && \
|
||||
VITE_API_URL=/api npm run build -- --outDir "$RSS2_HOME/frontend/dist")
|
||||
info " [OK] frontend"
|
||||
fi
|
||||
|
||||
# --- Workers Python ---
|
||||
info "Sincronizando workers Python..."
|
||||
rsync -a --delete "$REPO_ROOT/workers/" "$RSS2_HOME/src/workers/"
|
||||
cp "$REPO_ROOT/config/entity_config.json" "$RSS2_HOME/src/" 2>/dev/null || true
|
||||
info " [OK] workers Python"
|
||||
|
||||
chown -R rss2:rss2 "$RSS2_HOME/bin" "$RSS2_HOME/frontend/dist" "$RSS2_HOME/src"
|
||||
|
||||
# --- Restart servicios ---
|
||||
info "Reiniciando servicios..."
|
||||
GO_SERVICES=(rss2-backend rss2-ingestor rss2-scraper rss2-discovery rss2-wiki rss2-topics rss2-related rss2-qdrant-worker)
|
||||
PY_SERVICES=(rss2-langdetect rss2-translation-scheduler rss2-translator rss2-embeddings rss2-ner rss2-cluster rss2-categorizer)
|
||||
|
||||
for svc in "${GO_SERVICES[@]}" "${PY_SERVICES[@]}"; do
|
||||
systemctl is-active --quiet "$svc" && systemctl restart "$svc" && info " restarted $svc" || true
|
||||
done
|
||||
|
||||
systemctl reload nginx 2>/dev/null || true
|
||||
|
||||
info "Build completado."
|
||||
342
deploy/debian/check.sh
Executable file
342
deploy/debian/check.sh
Executable file
|
|
@ -0,0 +1,342 @@
|
|||
#!/usr/bin/env bash
|
||||
# =============================================================================
|
||||
# COCONEWS - Diagnóstico del sistema
|
||||
# Verifica que todos los servicios estén OK y muestra estado + sugerencias
|
||||
#
|
||||
# Uso:
|
||||
# bash deploy/debian/check.sh # diagnóstico completo
|
||||
# bash deploy/debian/check.sh --quick # solo servicios arriba/abajo
|
||||
# =============================================================================
|
||||
|
||||
RSS2_HOME="/opt/rss2"
|
||||
API_PORT="8080"
|
||||
QUICK=false
|
||||
[[ "${1:-}" == "--quick" ]] && QUICK=true
|
||||
|
||||
RED='\033[0;31m'; GREEN='\033[0;32m'; YELLOW='\033[1;33m'
|
||||
BLUE='\033[0;34m'; BOLD='\033[1m'; DIM='\033[2m'; NC='\033[0m'
|
||||
CYAN='\033[0;36m'
|
||||
|
||||
OK() { echo -e " ${GREEN}[✓]${NC} $*"; }
|
||||
FAIL() { echo -e " ${RED}[✗]${NC} $*"; }
|
||||
WARN() { echo -e " ${YELLOW}[!]${NC} $*"; }
|
||||
INFO() { echo -e " ${DIM} $*${NC}"; }
|
||||
HEAD() { echo -e "\n${BOLD}${CYAN}── $* ${NC}"; }
|
||||
FIX() { echo -e " ${YELLOW} → Fix:${NC} $*"; }
|
||||
|
||||
ERRORS=0
|
||||
WARNINGS=0
|
||||
|
||||
fail_with_fix() {
|
||||
FAIL "$1"
|
||||
shift
|
||||
for fix in "$@"; do FIX "$fix"; done
|
||||
(( ERRORS++ )) || true
|
||||
}
|
||||
|
||||
warn_with_fix() {
|
||||
WARN "$1"
|
||||
shift
|
||||
for fix in "$@"; do FIX "$fix"; done
|
||||
(( WARNINGS++ )) || true
|
||||
}
|
||||
|
||||
echo ""
|
||||
echo -e "${BOLD}╔═══════════════════════════════════════════╗${NC}"
|
||||
echo -e "${BOLD}║ COCONEWS · Diagnóstico del Sistema ║${NC}"
|
||||
echo -e "${BOLD}╚═══════════════════════════════════════════╝${NC}"
|
||||
echo -e "${DIM}$(date '+%Y-%m-%d %H:%M:%S') · $(hostname)${NC}"
|
||||
|
||||
# =============================================================================
|
||||
# 1. SERVICIOS SYSTEMD
|
||||
# =============================================================================
|
||||
HEAD "Servicios systemd"
|
||||
|
||||
GO_SERVICES=(
|
||||
"rss2-backend:API REST Go"
|
||||
"rss2-ingestor:Ingestor RSS"
|
||||
"rss2-scraper:Scraper HTML"
|
||||
"rss2-discovery:Discovery de feeds"
|
||||
"rss2-wiki:Wiki worker"
|
||||
"rss2-topics:Topics matcher"
|
||||
"rss2-related:Related news"
|
||||
"rss2-qdrant-worker:Qdrant sync"
|
||||
)
|
||||
PY_SERVICES=(
|
||||
"rss2-langdetect:Detección de idioma"
|
||||
"rss2-translation-scheduler:Scheduler traducción"
|
||||
"rss2-translator:Traductor NLLB-200"
|
||||
"rss2-embeddings:Embeddings ML"
|
||||
"rss2-ner:NER entidades"
|
||||
"rss2-cluster:Clustering eventos"
|
||||
"rss2-categorizer:Categorizador"
|
||||
)
|
||||
INFRA_SERVICES=(
|
||||
"rss2-qdrant:Vector DB Qdrant"
|
||||
"postgresql:PostgreSQL"
|
||||
"redis-server:Redis"
|
||||
"nginx:Nginx"
|
||||
)
|
||||
|
||||
check_service() {
|
||||
local svc="${1%%:*}"
|
||||
local name="${1##*:}"
|
||||
if ! command -v systemctl &>/dev/null; then
|
||||
WARN "$name (systemctl no disponible)"
|
||||
return
|
||||
fi
|
||||
local state
|
||||
state=$(systemctl is-active "$svc" 2>/dev/null || echo "unknown")
|
||||
case "$state" in
|
||||
active)
|
||||
OK "$name ($svc)"
|
||||
;;
|
||||
failed)
|
||||
fail_with_fix "$name ($svc) — FAILED" \
|
||||
"journalctl -u $svc -n 30 --no-pager" \
|
||||
"systemctl restart $svc"
|
||||
;;
|
||||
inactive)
|
||||
warn_with_fix "$name ($svc) — inactivo" \
|
||||
"systemctl start $svc" \
|
||||
"systemctl enable $svc"
|
||||
;;
|
||||
*)
|
||||
warn_with_fix "$name ($svc) — $state" \
|
||||
"systemctl status $svc"
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
echo -e "\n ${DIM}Infraestructura:${NC}"
|
||||
for svc in "${INFRA_SERVICES[@]}"; do check_service "$svc"; done
|
||||
|
||||
echo -e "\n ${DIM}Workers Go:${NC}"
|
||||
for svc in "${GO_SERVICES[@]}"; do check_service "$svc"; done
|
||||
|
||||
echo -e "\n ${DIM}Workers Python (ML):${NC}"
|
||||
for svc in "${PY_SERVICES[@]}"; do check_service "$svc"; done
|
||||
|
||||
[[ "$QUICK" == "true" ]] && {
|
||||
echo ""
|
||||
[[ "$ERRORS" -eq 0 && "$WARNINGS" -eq 0 ]] && \
|
||||
echo -e " ${GREEN}${BOLD}Todo OK${NC}" || \
|
||||
echo -e " ${RED}Errores: $ERRORS ${YELLOW}Advertencias: $WARNINGS${NC}"
|
||||
exit 0
|
||||
}
|
||||
|
||||
# =============================================================================
|
||||
# 2. CONECTIVIDAD
|
||||
# =============================================================================
|
||||
HEAD "Conectividad"
|
||||
|
||||
# PostgreSQL
|
||||
PG_ENV="$RSS2_HOME/.env"
|
||||
if [[ -f "$PG_ENV" ]]; then
|
||||
source "$PG_ENV" 2>/dev/null || true
|
||||
fi
|
||||
DB_NAME="${POSTGRES_DB:-rss}"
|
||||
DB_USER="${POSTGRES_USER:-rss}"
|
||||
|
||||
if pg_isready -q 2>/dev/null; then
|
||||
OK "PostgreSQL acepta conexiones"
|
||||
# Probar conexión con usuario rss2
|
||||
if sudo -u postgres psql -d "$DB_NAME" -c "SELECT 1" &>/dev/null 2>&1; then
|
||||
NEWS=$(sudo -u postgres psql -tq -d "$DB_NAME" \
|
||||
-c "SELECT COUNT(*) FROM noticias;" 2>/dev/null | tr -d ' \n' || echo "?")
|
||||
TRANS=$(sudo -u postgres psql -tq -d "$DB_NAME" \
|
||||
-c "SELECT COUNT(*) FROM traducciones WHERE status='done';" 2>/dev/null | tr -d ' \n' || echo "?")
|
||||
PEND=$(sudo -u postgres psql -tq -d "$DB_NAME" \
|
||||
-c "SELECT COUNT(*) FROM traducciones WHERE status='pending';" 2>/dev/null | tr -d ' \n' || echo "?")
|
||||
INFO "Base de datos: $DB_NAME"
|
||||
INFO "Noticias: $NEWS | Traducciones hechas: $TRANS | Pendientes: $PEND"
|
||||
else
|
||||
warn_with_fix "No se puede conectar a la BD '$DB_NAME' con usuario '$DB_USER'" \
|
||||
"sudo -u postgres psql -c \"\\du\" # listar usuarios" \
|
||||
"Revisa DB_USER y DB_PASS en $RSS2_HOME/.env"
|
||||
fi
|
||||
else
|
||||
fail_with_fix "PostgreSQL no responde" \
|
||||
"sudo systemctl start postgresql" \
|
||||
"sudo journalctl -u postgresql -n 20 --no-pager" \
|
||||
"sudo pg_ctlcluster 16 main status"
|
||||
fi
|
||||
|
||||
# Redis
|
||||
REDIS_PASS="${REDIS_PASSWORD:-}"
|
||||
REDIS_AUTH=""
|
||||
[[ -n "$REDIS_PASS" ]] && REDIS_AUTH="-a $REDIS_PASS"
|
||||
if redis-cli $REDIS_AUTH ping 2>/dev/null | grep -q PONG; then
|
||||
REDIS_MEM=$(redis-cli $REDIS_AUTH info memory 2>/dev/null | grep used_memory_human | cut -d: -f2 | tr -d '\r' || echo "?")
|
||||
OK "Redis responde (memoria usada: ${REDIS_MEM})"
|
||||
else
|
||||
fail_with_fix "Redis no responde" \
|
||||
"sudo systemctl start redis-server" \
|
||||
"redis-cli ping # sin auth" \
|
||||
"Verifica REDIS_PASSWORD en $RSS2_HOME/.env"
|
||||
fi
|
||||
|
||||
# Qdrant
|
||||
if curl -sf "http://127.0.0.1:6333/healthz" &>/dev/null; then
|
||||
QDRANT_VER=$(curl -sf "http://127.0.0.1:6333/" 2>/dev/null | python3 -c "import sys,json; d=json.load(sys.stdin); print(d.get('version','?'))" 2>/dev/null || echo "?")
|
||||
OK "Qdrant responde (v${QDRANT_VER})"
|
||||
else
|
||||
warn_with_fix "Qdrant no responde en :6333" \
|
||||
"systemctl start rss2-qdrant" \
|
||||
"journalctl -u rss2-qdrant -n 20 --no-pager" \
|
||||
"Búsqueda semántica no funcionará hasta que Qdrant esté activo"
|
||||
fi
|
||||
|
||||
# API Backend Go
|
||||
if curl -sf "http://127.0.0.1:${API_PORT}/api/stats" &>/dev/null; then
|
||||
STATS=$(curl -sf "http://127.0.0.1:${API_PORT}/api/stats" 2>/dev/null | \
|
||||
python3 -c "import sys,json; d=json.load(sys.stdin); print(f\"noticias={d.get('total_news','?')} feeds={d.get('total_feeds','?')}\")" 2>/dev/null || echo "")
|
||||
OK "API backend responde (:${API_PORT}) — $STATS"
|
||||
else
|
||||
fail_with_fix "API backend no responde en :${API_PORT}" \
|
||||
"systemctl restart rss2-backend" \
|
||||
"journalctl -u rss2-backend -n 30 --no-pager" \
|
||||
"curl http://127.0.0.1:${API_PORT}/api/stats # prueba manual"
|
||||
fi
|
||||
|
||||
# Nginx
|
||||
if curl -sf "http://127.0.0.1:8001/health" &>/dev/null; then
|
||||
OK "Nginx responde (:8001)"
|
||||
else
|
||||
fail_with_fix "Nginx no responde en :8001" \
|
||||
"nginx -t # verificar config" \
|
||||
"systemctl restart nginx" \
|
||||
"journalctl -u nginx -n 20 --no-pager"
|
||||
fi
|
||||
|
||||
# =============================================================================
|
||||
# 3. BINARIOS GO
|
||||
# =============================================================================
|
||||
HEAD "Binarios compilados"
|
||||
|
||||
BINS=(server ingestor scraper discovery wiki_worker topics related qdrant_worker)
|
||||
for bin in "${BINS[@]}"; do
|
||||
BIN_PATH="$RSS2_HOME/bin/$bin"
|
||||
if [[ -x "$BIN_PATH" ]]; then
|
||||
SIZE=$(du -sh "$BIN_PATH" 2>/dev/null | cut -f1)
|
||||
OK "$bin (${SIZE})"
|
||||
else
|
||||
fail_with_fix "$bin no encontrado o sin permisos de ejecución en $BIN_PATH" \
|
||||
"sudo bash deploy/debian/build.sh # recompila todos los binarios"
|
||||
fi
|
||||
done
|
||||
|
||||
# =============================================================================
|
||||
# 4. MODELOS ML
|
||||
# =============================================================================
|
||||
HEAD "Modelos ML"
|
||||
|
||||
# NLLB-200 CTranslate2
|
||||
if [[ -d "$RSS2_HOME/models/nllb-ct2" ]] && \
|
||||
[[ -f "$RSS2_HOME/models/nllb-ct2/model.bin" ]]; then
|
||||
SIZE=$(du -sh "$RSS2_HOME/models/nllb-ct2" 2>/dev/null | cut -f1)
|
||||
OK "NLLB-200 CTranslate2 (${SIZE})"
|
||||
else
|
||||
warn_with_fix "Modelo NLLB-200 no encontrado en $RSS2_HOME/models/nllb-ct2" \
|
||||
"El traductor no funcionará hasta que esté el modelo" \
|
||||
"Convierte con: ver sección 3 de DEPLOY_DEBIAN.md"
|
||||
fi
|
||||
|
||||
# spaCy
|
||||
if "$RSS2_HOME/venv/bin/python" -c "import spacy; spacy.load('es_core_news_lg')" &>/dev/null 2>&1; then
|
||||
OK "spaCy es_core_news_lg"
|
||||
else
|
||||
warn_with_fix "Modelo spaCy es_core_news_lg no disponible" \
|
||||
"$RSS2_HOME/venv/bin/python -m spacy download es_core_news_lg" \
|
||||
"El worker NER no funcionará hasta instalarlo"
|
||||
fi
|
||||
|
||||
# sentence-transformers (embeddings)
|
||||
if "$RSS2_HOME/venv/bin/python" -c "from sentence_transformers import SentenceTransformer" &>/dev/null 2>&1; then
|
||||
OK "sentence-transformers disponible"
|
||||
else
|
||||
warn_with_fix "sentence-transformers no instalado" \
|
||||
"$RSS2_HOME/venv/bin/pip install sentence-transformers==3.0.1" \
|
||||
"Los embeddings y búsqueda semántica no funcionarán"
|
||||
fi
|
||||
|
||||
# ctranslate2
|
||||
if "$RSS2_HOME/venv/bin/python" -c "import ctranslate2" &>/dev/null 2>&1; then
|
||||
OK "ctranslate2 disponible"
|
||||
else
|
||||
warn_with_fix "ctranslate2 no instalado" \
|
||||
"$RSS2_HOME/venv/bin/pip install ctranslate2>=4.0.0" \
|
||||
"La traducción NLLB-200 no funcionará"
|
||||
fi
|
||||
|
||||
# =============================================================================
|
||||
# 5. DISCO Y PERMISOS
|
||||
# =============================================================================
|
||||
HEAD "Disco y permisos"
|
||||
|
||||
# Espacio en disco
|
||||
DISK_FREE=$(df -h "$RSS2_HOME" 2>/dev/null | tail -1 | awk '{print $4}')
|
||||
DISK_PCT=$(df "$RSS2_HOME" 2>/dev/null | tail -1 | awk '{print $5}' | tr -d '%')
|
||||
if [[ "${DISK_PCT:-0}" -gt 90 ]]; then
|
||||
fail_with_fix "Disco al ${DISK_PCT}% — quedan solo ${DISK_FREE}" \
|
||||
"du -sh $RSS2_HOME/* | sort -rh | head -10 # ver qué ocupa más" \
|
||||
"Los modelos ML necesitan ~5 GB libres"
|
||||
elif [[ "${DISK_PCT:-0}" -gt 75 ]]; then
|
||||
warn_with_fix "Disco al ${DISK_PCT}% (libre: ${DISK_FREE})" \
|
||||
"Revisa el espacio antes de descargar modelos ML"
|
||||
else
|
||||
OK "Disco: ${DISK_PCT}% usado, ${DISK_FREE} libres"
|
||||
fi
|
||||
|
||||
# Permisos de /opt/rss2
|
||||
if [[ -d "$RSS2_HOME" ]]; then
|
||||
OWNER=$(stat -c '%U' "$RSS2_HOME" 2>/dev/null || echo "?")
|
||||
if [[ "$OWNER" == "rss2" ]]; then
|
||||
OK "Propietario de $RSS2_HOME: rss2"
|
||||
else
|
||||
warn_with_fix "Propietario de $RSS2_HOME es '$OWNER', debería ser 'rss2'" \
|
||||
"sudo chown -R rss2:rss2 $RSS2_HOME"
|
||||
fi
|
||||
fi
|
||||
|
||||
# .env
|
||||
if [[ -f "$RSS2_HOME/.env" ]]; then
|
||||
ENV_PERMS=$(stat -c '%a' "$RSS2_HOME/.env" 2>/dev/null || echo "?")
|
||||
if [[ "$ENV_PERMS" == "600" ]]; then
|
||||
OK ".env con permisos 600 (correcto)"
|
||||
else
|
||||
warn_with_fix ".env con permisos ${ENV_PERMS} (debería ser 600)" \
|
||||
"chmod 600 $RSS2_HOME/.env"
|
||||
fi
|
||||
# Verificar que no quedan valores por defecto peligrosos
|
||||
if grep -q "CAMBIA_ESTO\|changeme\|change_this" "$RSS2_HOME/.env" 2>/dev/null; then
|
||||
warn_with_fix ".env tiene valores por defecto sin cambiar" \
|
||||
"nano $RSS2_HOME/.env # cambia POSTGRES_PASSWORD, REDIS_PASSWORD, SECRET_KEY"
|
||||
fi
|
||||
else
|
||||
fail_with_fix ".env no encontrado en $RSS2_HOME/.env" \
|
||||
"cp deploy/debian/env.example $RSS2_HOME/.env" \
|
||||
"nano $RSS2_HOME/.env # edita contraseñas"
|
||||
fi
|
||||
|
||||
# =============================================================================
|
||||
# RESUMEN FINAL
|
||||
# =============================================================================
|
||||
echo ""
|
||||
echo -e "${BOLD}══════════════════════════════════════════════${NC}"
|
||||
if [[ "$ERRORS" -eq 0 && "$WARNINGS" -eq 0 ]]; then
|
||||
echo -e " ${GREEN}${BOLD}Sistema OK — todo funcionando correctamente${NC}"
|
||||
elif [[ "$ERRORS" -eq 0 ]]; then
|
||||
echo -e " ${YELLOW}${BOLD}Sistema funcional con $WARNINGS advertencia(s)${NC}"
|
||||
echo -e " ${DIM}Las advertencias no bloquean el funcionamiento básico${NC}"
|
||||
else
|
||||
echo -e " ${RED}${BOLD}$ERRORS error(es) encontrado(s)${YELLOW} $WARNINGS advertencia(s)${NC}"
|
||||
echo -e " ${DIM}Sigue los fixes indicados arriba para resolverlos${NC}"
|
||||
fi
|
||||
echo -e "${BOLD}══════════════════════════════════════════════${NC}"
|
||||
echo ""
|
||||
echo -e " ${DIM}Logs de servicios: journalctl -u rss2-backend -f${NC}"
|
||||
echo -e " ${DIM}Diagnóstico rápido: bash deploy/debian/check.sh --quick${NC}"
|
||||
echo ""
|
||||
|
||||
exit $(( ERRORS > 0 ? 1 : 0 ))
|
||||
112
deploy/debian/env.example
Normal file
112
deploy/debian/env.example
Normal file
|
|
@ -0,0 +1,112 @@
|
|||
# =============================================================================
|
||||
# RSS2 - Variables de entorno para despliegue Debian nativo
|
||||
# Copiar a /opt/rss2/.env y editar valores antes de instalar
|
||||
# =============================================================================
|
||||
|
||||
# --- PostgreSQL ---
|
||||
POSTGRES_DB=rss
|
||||
POSTGRES_USER=rss
|
||||
POSTGRES_PASSWORD=CAMBIA_ESTO_postgres_password
|
||||
|
||||
# Usadas por workers Go (equivalente a DATABASE_URL)
|
||||
DB_HOST=127.0.0.1
|
||||
DB_PORT=5432
|
||||
DB_NAME=rss
|
||||
DB_USER=rss
|
||||
DB_PASS=CAMBIA_ESTO_postgres_password
|
||||
|
||||
# URL completa para backend API Go
|
||||
DATABASE_URL=postgres://rss:CAMBIA_ESTO_postgres_password@127.0.0.1:5432/rss?sslmode=disable
|
||||
|
||||
# --- Redis ---
|
||||
REDIS_PASSWORD=CAMBIA_ESTO_redis_password
|
||||
REDIS_URL=redis://:CAMBIA_ESTO_redis_password@127.0.0.1:6379
|
||||
|
||||
# --- JWT Secret (minimo 32 caracteres, aleatorio) ---
|
||||
SECRET_KEY=CAMBIA_ESTO_jwt_secret_muy_largo_y_aleatorio
|
||||
|
||||
# --- Backend API ---
|
||||
SERVER_PORT=8080
|
||||
|
||||
# --- Zona horaria ---
|
||||
TZ=Europe/Madrid
|
||||
|
||||
# --- HuggingFace cache (modelos ML) ---
|
||||
HF_HOME=/opt/rss2/hf_cache
|
||||
|
||||
# --- Endpoints ML opcionales (solo si corres Ollama o LibreTranslate por separado) ---
|
||||
# Los workers Python van directo a BD; estos endpoints solo los usa el backend para
|
||||
# llamadas on-demand desde el panel admin (NER, traduccion manual, etc.)
|
||||
TRANSLATION_URL=http://127.0.0.1:7790
|
||||
OLLAMA_URL=http://127.0.0.1:11434
|
||||
SPACY_URL=http://127.0.0.1:8000
|
||||
|
||||
# --- Qdrant (local, sin Docker) ---
|
||||
QDRANT_HOST=127.0.0.1
|
||||
QDRANT_PORT=6333
|
||||
QDRANT_COLLECTION=news_vectors
|
||||
|
||||
# --- Translator (NLLB-200 via CTranslate2) ---
|
||||
TARGET_LANGS=es
|
||||
TRANSLATOR_BATCH=32
|
||||
CT2_MODEL_PATH=/opt/rss2/models/nllb-ct2
|
||||
CT2_DEVICE=cpu
|
||||
CT2_COMPUTE_TYPE=int8
|
||||
UNIVERSAL_MODEL=facebook/nllb-200-distilled-600M
|
||||
|
||||
# --- Embeddings ---
|
||||
EMB_MODEL=sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2
|
||||
EMB_BATCH=64
|
||||
EMB_SLEEP_IDLE=5
|
||||
EMB_LANGS=es
|
||||
EMB_LIMIT=1000
|
||||
DEVICE=cpu
|
||||
|
||||
# --- NER ---
|
||||
NER_LANG=es
|
||||
NER_BATCH=64
|
||||
|
||||
# --- Ingestor RSS ---
|
||||
RSS_MAX_WORKERS=100
|
||||
RSS_POKE_INTERVAL_MIN=15
|
||||
|
||||
# --- Scraper ---
|
||||
SCRAPER_SLEEP=60
|
||||
SCRAPER_BATCH=10
|
||||
|
||||
# --- Discovery ---
|
||||
DISCOVERY_INTERVAL=900
|
||||
DISCOVERY_BATCH=10
|
||||
MAX_FEEDS_PER_URL=5
|
||||
|
||||
# --- Wiki Worker ---
|
||||
WIKI_SLEEP=10
|
||||
WIKI_IMAGES_PATH=/opt/rss2/data/wiki_images
|
||||
|
||||
# --- Topics ---
|
||||
TOPICS_SLEEP=10
|
||||
TOPICS_BATCH=500
|
||||
|
||||
# --- Related ---
|
||||
RELATED_SLEEP=10
|
||||
RELATED_BATCH=200
|
||||
RELATED_TOPK=10
|
||||
|
||||
# --- Cluster ---
|
||||
EVENT_DIST_THRESHOLD=0.35
|
||||
|
||||
# --- Categorizer ---
|
||||
CATEGORIZER_BATCH_SIZE=10
|
||||
CATEGORIZER_SLEEP_IDLE=5
|
||||
|
||||
# --- Scheduler traduccion ---
|
||||
SCHEDULER_BATCH=1000
|
||||
SCHEDULER_SLEEP=30
|
||||
|
||||
# --- Lang Detect ---
|
||||
LANG_DETECT_SLEEP=60
|
||||
LANG_DETECT_BATCH=1000
|
||||
|
||||
# --- Qdrant Worker ---
|
||||
QDRANT_SLEEP=30
|
||||
QDRANT_BATCH=100
|
||||
309
deploy/debian/install.sh
Executable file
309
deploy/debian/install.sh
Executable file
|
|
@ -0,0 +1,309 @@
|
|||
#!/usr/bin/env bash
|
||||
# =============================================================================
|
||||
# RSS2 - Instalacion en Debian (sin Docker)
|
||||
# Ejecutar como root: bash install.sh
|
||||
# =============================================================================
|
||||
set -euo pipefail
|
||||
|
||||
RSS2_USER="rss2"
|
||||
RSS2_HOME="/opt/rss2"
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
|
||||
|
||||
RED='\033[0;31m'; GREEN='\033[0;32m'; YELLOW='\033[1;33m'; NC='\033[0m'
|
||||
info() { echo -e "${GREEN}[INFO]${NC} $*"; }
|
||||
warn() { echo -e "${YELLOW}[WARN]${NC} $*"; }
|
||||
error() { echo -e "${RED}[ERROR]${NC} $*"; exit 1; }
|
||||
|
||||
[[ "$EUID" -ne 0 ]] && error "Ejecutar como root: sudo bash install.sh"
|
||||
|
||||
# =============================================================================
|
||||
# 1. DEPENDENCIAS DEL SISTEMA
|
||||
# =============================================================================
|
||||
info "Instalando dependencias del sistema..."
|
||||
apt-get update -qq
|
||||
apt-get install -y --no-install-recommends \
|
||||
curl wget git build-essential \
|
||||
postgresql postgresql-client \
|
||||
redis-server \
|
||||
nginx \
|
||||
python3 python3-pip python3-venv python3-dev \
|
||||
nodejs npm \
|
||||
ca-certificates tzdata \
|
||||
libpq-dev
|
||||
|
||||
# Go (rss-ingestor-go requiere Go 1.25)
|
||||
_need_go=false
|
||||
if ! command -v go &>/dev/null; then
|
||||
_need_go=true
|
||||
else
|
||||
_gover=$(go version | awk '{print $3}' | tr -d 'go')
|
||||
IFS='.' read -ra _gv <<< "$_gover"
|
||||
if [[ "${_gv[0]:-0}" -lt 1 ]] || [[ "${_gv[0]:-0}" -eq 1 && "${_gv[1]:-0}" -lt 25 ]]; then
|
||||
_need_go=true
|
||||
fi
|
||||
fi
|
||||
if [[ "$_need_go" == "true" ]]; then
|
||||
info "Instalando Go 1.25..."
|
||||
GO_VERSION="1.25.0"
|
||||
ARCH=$(dpkg --print-architecture)
|
||||
case "$ARCH" in
|
||||
amd64) GO_ARCH="amd64" ;;
|
||||
arm64) GO_ARCH="arm64" ;;
|
||||
*) error "Arquitectura no soportada: $ARCH" ;;
|
||||
esac
|
||||
curl -fsSL "https://go.dev/dl/go${GO_VERSION}.linux-${GO_ARCH}.tar.gz" -o /tmp/go.tar.gz
|
||||
rm -rf /usr/local/go
|
||||
tar -C /usr/local -xzf /tmp/go.tar.gz
|
||||
echo 'export PATH=$PATH:/usr/local/go/bin' > /etc/profile.d/go.sh
|
||||
export PATH=$PATH:/usr/local/go/bin
|
||||
rm /tmp/go.tar.gz
|
||||
fi
|
||||
info "Go: $(go version)"
|
||||
|
||||
# Qdrant (binario oficial)
|
||||
if [[ ! -f "$RSS2_HOME/qdrant/qdrant" ]]; then
|
||||
info "Descargando Qdrant..."
|
||||
QDRANT_VERSION="v1.12.1"
|
||||
ARCH=$(dpkg --print-architecture)
|
||||
case "$ARCH" in
|
||||
amd64) QDRANT_ARCH="x86_64-unknown-linux-musl" ;;
|
||||
arm64) QDRANT_ARCH="aarch64-unknown-linux-musl" ;;
|
||||
*) error "Arquitectura no soportada para Qdrant: $ARCH" ;;
|
||||
esac
|
||||
mkdir -p "$RSS2_HOME/qdrant"
|
||||
curl -fsSL "https://github.com/qdrant/qdrant/releases/download/${QDRANT_VERSION}/qdrant-${QDRANT_ARCH}.tar.gz" \
|
||||
-o /tmp/qdrant.tar.gz
|
||||
tar -C "$RSS2_HOME/qdrant" -xzf /tmp/qdrant.tar.gz
|
||||
chmod +x "$RSS2_HOME/qdrant/qdrant"
|
||||
rm /tmp/qdrant.tar.gz
|
||||
fi
|
||||
|
||||
# =============================================================================
|
||||
# 2. USUARIO Y DIRECTORIOS
|
||||
# =============================================================================
|
||||
info "Creando usuario $RSS2_USER y directorios..."
|
||||
id "$RSS2_USER" &>/dev/null || useradd -r -m -d "$RSS2_HOME" -s /bin/bash "$RSS2_USER"
|
||||
|
||||
mkdir -p \
|
||||
"$RSS2_HOME/bin" \
|
||||
"$RSS2_HOME/src" \
|
||||
"$RSS2_HOME/data/wiki_images" \
|
||||
"$RSS2_HOME/data/qdrant_storage" \
|
||||
"$RSS2_HOME/hf_cache" \
|
||||
"$RSS2_HOME/models" \
|
||||
"$RSS2_HOME/frontend/dist" \
|
||||
"$RSS2_HOME/logs"
|
||||
|
||||
# =============================================================================
|
||||
# 3. CONFIGURACION ENTORNO
|
||||
# =============================================================================
|
||||
if [[ ! -f "$RSS2_HOME/.env" ]]; then
|
||||
if [[ -f "$SCRIPT_DIR/env.example" ]]; then
|
||||
cp "$SCRIPT_DIR/env.example" "$RSS2_HOME/.env"
|
||||
warn "Copia env.example en $RSS2_HOME/.env - EDITA LAS CONTRASENAS antes de continuar"
|
||||
warn "Presiona Enter cuando hayas editado el .env, o Ctrl+C para salir"
|
||||
read -r
|
||||
else
|
||||
error "No se encontro env.example en $SCRIPT_DIR"
|
||||
fi
|
||||
fi
|
||||
|
||||
# =============================================================================
|
||||
# 4. POSTGRESQL
|
||||
# =============================================================================
|
||||
info "Configurando PostgreSQL..."
|
||||
source "$RSS2_HOME/.env" 2>/dev/null || true
|
||||
|
||||
DB_NAME="${POSTGRES_DB:-rss}"
|
||||
DB_USER="${POSTGRES_USER:-rss}"
|
||||
DB_PASS="${POSTGRES_PASSWORD:-changeme}"
|
||||
|
||||
systemctl enable --now postgresql
|
||||
|
||||
# Crear usuario y base de datos si no existen
|
||||
sudo -u postgres psql -tc "SELECT 1 FROM pg_roles WHERE rolname='$DB_USER'" | grep -q 1 || \
|
||||
sudo -u postgres psql -c "CREATE USER $DB_USER WITH PASSWORD '$DB_PASS';"
|
||||
|
||||
sudo -u postgres psql -tc "SELECT 1 FROM pg_database WHERE datname='$DB_NAME'" | grep -q 1 || \
|
||||
sudo -u postgres createdb -O "$DB_USER" "$DB_NAME"
|
||||
|
||||
# Ejecutar migraciones SQL
|
||||
if [[ -d "$REPO_ROOT/migrations" ]]; then
|
||||
info "Ejecutando migraciones..."
|
||||
for sql_file in "$REPO_ROOT/migrations"/*.sql; do
|
||||
[[ -f "$sql_file" ]] || continue
|
||||
info " Aplicando $(basename "$sql_file")..."
|
||||
sudo -u postgres psql -d "$DB_NAME" -f "$sql_file" 2>/dev/null || warn " (ya aplicada o error ignorado)"
|
||||
done
|
||||
fi
|
||||
|
||||
# Ejecutar init-db scripts (schema inicial)
|
||||
if [[ -d "$REPO_ROOT/init-db" ]]; then
|
||||
info "Ejecutando scripts de init-db..."
|
||||
for sql_file in "$REPO_ROOT/init-db"/*.sql; do
|
||||
[[ -f "$sql_file" ]] || continue
|
||||
info " $(basename "$sql_file")..."
|
||||
sudo -u postgres psql -d "$DB_NAME" -f "$sql_file" 2>/dev/null || warn " (ya aplicada o error ignorado)"
|
||||
done
|
||||
fi
|
||||
|
||||
# =============================================================================
|
||||
# 5. REDIS
|
||||
# =============================================================================
|
||||
info "Configurando Redis..."
|
||||
REDIS_PASS="${REDIS_PASSWORD:-changeme_redis}"
|
||||
|
||||
# Agregar autenticacion y limites de memoria a redis.conf
|
||||
REDIS_CONF="/etc/redis/redis.conf"
|
||||
grep -q "requirepass $REDIS_PASS" "$REDIS_CONF" 2>/dev/null || {
|
||||
echo "requirepass $REDIS_PASS" >> "$REDIS_CONF"
|
||||
echo "maxmemory 512mb" >> "$REDIS_CONF"
|
||||
echo "maxmemory-policy allkeys-lru" >> "$REDIS_CONF"
|
||||
echo "appendonly yes" >> "$REDIS_CONF"
|
||||
}
|
||||
systemctl enable --now redis-server
|
||||
|
||||
# =============================================================================
|
||||
# 6. PYTHON VIRTUALENV + DEPENDENCIAS ML
|
||||
# =============================================================================
|
||||
info "Creando virtualenv Python y instalando dependencias..."
|
||||
python3 -m venv "$RSS2_HOME/venv"
|
||||
"$RSS2_HOME/venv/bin/pip" install --upgrade pip -q
|
||||
|
||||
if [[ -f "$REPO_ROOT/requirements.txt" ]]; then
|
||||
"$RSS2_HOME/venv/bin/pip" install -r "$REPO_ROOT/requirements.txt" -q
|
||||
fi
|
||||
|
||||
# spaCy modelo en español
|
||||
"$RSS2_HOME/venv/bin/python" -m spacy download es_core_news_lg 2>/dev/null || \
|
||||
warn "spaCy model es_core_news_lg no se pudo descargar, hazlo manualmente"
|
||||
|
||||
# Copiar workers Python al directorio de trabajo
|
||||
info "Copiando workers Python..."
|
||||
rsync -a --delete "$REPO_ROOT/workers/" "$RSS2_HOME/src/workers/"
|
||||
cp "$REPO_ROOT/config/entity_config.json" "$RSS2_HOME/src/" 2>/dev/null || true
|
||||
|
||||
# =============================================================================
|
||||
# 7. COMPILAR GO (backend + workers)
|
||||
# =============================================================================
|
||||
info "Compilando binarios Go..."
|
||||
export PATH=$PATH:/usr/local/go/bin
|
||||
export GOPATH=/tmp/go-build-rss2
|
||||
|
||||
# Backend API
|
||||
if [[ -d "$REPO_ROOT/backend" ]]; then
|
||||
(cd "$REPO_ROOT/backend" && \
|
||||
CGO_ENABLED=0 GOOS=linux go build -buildvcs=false -o "$RSS2_HOME/bin/server" ./cmd/server && \
|
||||
info " [OK] server") || warn " [FAIL] server"
|
||||
for cmd in scraper discovery wiki_worker topics related; do
|
||||
[[ -d "$REPO_ROOT/backend/cmd/$cmd" ]] || continue
|
||||
(cd "$REPO_ROOT/backend" && \
|
||||
CGO_ENABLED=0 GOOS=linux go build -buildvcs=false -o "$RSS2_HOME/bin/$cmd" "./cmd/$cmd" && \
|
||||
info " [OK] $cmd") || warn " [FAIL] $cmd"
|
||||
done
|
||||
# qdrant worker: output como qdrant_worker para coincidir con el service
|
||||
[[ -d "$REPO_ROOT/backend/cmd/qdrant" ]] && \
|
||||
(cd "$REPO_ROOT/backend" && \
|
||||
CGO_ENABLED=0 GOOS=linux go build -buildvcs=false -o "$RSS2_HOME/bin/qdrant_worker" "./cmd/qdrant" && \
|
||||
info " [OK] qdrant_worker") || warn " [FAIL] qdrant_worker"
|
||||
fi
|
||||
|
||||
# RSS Ingestor Go (repo separado, requiere Go 1.25)
|
||||
if [[ -d "$REPO_ROOT/rss-ingestor-go" ]]; then
|
||||
(cd "$REPO_ROOT/rss-ingestor-go" && \
|
||||
GOTOOLCHAIN=local CGO_ENABLED=0 GOOS=linux go build -buildvcs=false -o "$RSS2_HOME/bin/ingestor" . && \
|
||||
info " [OK] ingestor") || warn " [FAIL] ingestor"
|
||||
fi
|
||||
|
||||
# =============================================================================
|
||||
# 8. FRONTEND REACT
|
||||
# =============================================================================
|
||||
info "Compilando frontend React..."
|
||||
if [[ -d "$REPO_ROOT/frontend" ]]; then
|
||||
(cd "$REPO_ROOT/frontend" && \
|
||||
npm install --silent && \
|
||||
VITE_API_URL=/api npm run build -- --outDir "$RSS2_HOME/frontend/dist" && \
|
||||
info " [OK] frontend compilado") || warn " [FAIL] frontend"
|
||||
fi
|
||||
|
||||
# =============================================================================
|
||||
# 9. NGINX
|
||||
# =============================================================================
|
||||
info "Configurando Nginx..."
|
||||
cp "$SCRIPT_DIR/nginx.conf" /etc/nginx/nginx.conf
|
||||
nginx -t && systemctl enable --now nginx && systemctl reload nginx
|
||||
|
||||
# =============================================================================
|
||||
# 10. SYSTEMD SERVICES
|
||||
# =============================================================================
|
||||
info "Instalando servicios systemd..."
|
||||
SERVICES=(
|
||||
rss2-qdrant
|
||||
rss2-backend
|
||||
rss2-ingestor
|
||||
rss2-scraper
|
||||
rss2-discovery
|
||||
rss2-wiki
|
||||
rss2-topics
|
||||
rss2-related
|
||||
rss2-qdrant-worker
|
||||
rss2-langdetect
|
||||
rss2-translation-scheduler
|
||||
rss2-translator
|
||||
rss2-embeddings
|
||||
rss2-ner
|
||||
rss2-cluster
|
||||
rss2-categorizer
|
||||
)
|
||||
|
||||
for svc in "${SERVICES[@]}"; do
|
||||
svc_file="$SCRIPT_DIR/systemd/${svc}.service"
|
||||
if [[ -f "$svc_file" ]]; then
|
||||
cp "$svc_file" "/etc/systemd/system/${svc}.service"
|
||||
else
|
||||
warn "No se encontro $svc_file"
|
||||
fi
|
||||
done
|
||||
|
||||
systemctl daemon-reload
|
||||
|
||||
for svc in "${SERVICES[@]}"; do
|
||||
systemctl enable "$svc" 2>/dev/null || true
|
||||
done
|
||||
|
||||
# =============================================================================
|
||||
# 11. PERMISOS FINALES
|
||||
# =============================================================================
|
||||
info "Ajustando permisos..."
|
||||
chown -R "$RSS2_USER:$RSS2_USER" "$RSS2_HOME"
|
||||
chmod 600 "$RSS2_HOME/.env"
|
||||
|
||||
# =============================================================================
|
||||
# 12. ARRANCAR SERVICIOS
|
||||
# =============================================================================
|
||||
info "Arrancando servicios..."
|
||||
# Infraestructura primero
|
||||
systemctl start rss2-qdrant
|
||||
sleep 3
|
||||
|
||||
# API y workers Go
|
||||
for svc in rss2-backend rss2-ingestor rss2-scraper rss2-discovery rss2-wiki rss2-topics rss2-related rss2-qdrant-worker; do
|
||||
systemctl start "$svc" || warn "No se pudo arrancar $svc"
|
||||
done
|
||||
|
||||
# Workers Python (modelos pesados, arrancan despues)
|
||||
for svc in rss2-langdetect rss2-translation-scheduler rss2-translator rss2-embeddings rss2-ner rss2-cluster rss2-categorizer; do
|
||||
systemctl start "$svc" || warn "No se pudo arrancar $svc"
|
||||
done
|
||||
|
||||
# =============================================================================
|
||||
echo ""
|
||||
info "============================================="
|
||||
info " RSS2 instalado en $RSS2_HOME"
|
||||
info " Acceder en: http://$(hostname -I | awk '{print $1}'):8001"
|
||||
info ""
|
||||
info " Ver logs: journalctl -u rss2-backend -f"
|
||||
info " Ver estado: systemctl status rss2-backend"
|
||||
info " Editar env: nano $RSS2_HOME/.env"
|
||||
info "============================================="
|
||||
|
|
@ -1,7 +1,7 @@
|
|||
user nginx;
|
||||
user www-data;
|
||||
worker_processes auto;
|
||||
error_log /var/log/nginx/error.log warn;
|
||||
pid /var/run/nginx.pid;
|
||||
pid /run/nginx.pid;
|
||||
|
||||
events {
|
||||
worker_connections 2048;
|
||||
|
|
@ -33,40 +33,36 @@ http {
|
|||
application/json application/javascript
|
||||
application/xml text/xml;
|
||||
|
||||
# Upstream for Go API
|
||||
# Go API backend (proceso nativo en localhost)
|
||||
upstream api_backend {
|
||||
server backend-go:8080;
|
||||
server 127.0.0.1:8080;
|
||||
keepalive 32;
|
||||
}
|
||||
|
||||
# Upstream for React Frontend
|
||||
upstream frontend {
|
||||
server rss2_frontend:80;
|
||||
keepalive 16;
|
||||
}
|
||||
|
||||
server {
|
||||
listen 80;
|
||||
listen 8001;
|
||||
server_name _;
|
||||
|
||||
client_body_timeout 60s;
|
||||
client_header_timeout 60s;
|
||||
send_timeout 300s;
|
||||
|
||||
# Serve React Frontend
|
||||
# Frontend React (archivos estaticos compilados)
|
||||
root /opt/rss2/frontend/dist;
|
||||
index index.html;
|
||||
|
||||
location / {
|
||||
proxy_pass http://frontend;
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Upgrade $http_upgrade;
|
||||
proxy_set_header Connection 'upgrade';
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
proxy_cache_bypass $http_upgrade;
|
||||
try_files $uri $uri/ /index.html;
|
||||
}
|
||||
|
||||
# Proxy to Go API
|
||||
# Imagenes Wikipedia servidas directamente
|
||||
location /wiki-images/ {
|
||||
alias /opt/rss2/data/wiki_images/;
|
||||
expires 7d;
|
||||
add_header Cache-Control "public, immutable";
|
||||
}
|
||||
|
||||
# Proxy al API Go
|
||||
location /api/ {
|
||||
proxy_pass http://api_backend/api/;
|
||||
proxy_http_version 1.1;
|
||||
|
|
@ -81,13 +77,11 @@ http {
|
|||
proxy_read_timeout 300s;
|
||||
}
|
||||
|
||||
# Health check
|
||||
location /health {
|
||||
access_log off;
|
||||
return 200 "ok";
|
||||
}
|
||||
|
||||
# Block sensitive files
|
||||
location ~ /\. {
|
||||
deny all;
|
||||
access_log off;
|
||||
327
deploy/debian/prerequisites.sh
Executable file
327
deploy/debian/prerequisites.sh
Executable file
|
|
@ -0,0 +1,327 @@
|
|||
#!/usr/bin/env bash
|
||||
# =============================================================================
|
||||
# COCONEWS - Instalacion de prerequisites en Debian 12 / Ubuntu 22.04+
|
||||
# Ejecutar ANTES de install.sh
|
||||
# Uso: sudo bash prerequisites.sh
|
||||
# =============================================================================
|
||||
set -euo pipefail
|
||||
|
||||
RED='\033[0;31m'; GREEN='\033[0;32m'; YELLOW='\033[1;33m'; BLUE='\033[0;34m'; NC='\033[0m'
|
||||
info() { echo -e "${GREEN}[OK]${NC} $*"; }
|
||||
step() { echo -e "${BLUE}[-->]${NC} $*"; }
|
||||
warn() { echo -e "${YELLOW}[WARN]${NC} $*"; }
|
||||
error() { echo -e "${RED}[ERROR]${NC} $*"; exit 1; }
|
||||
|
||||
[[ "$EUID" -ne 0 ]] && error "Ejecutar como root: sudo bash prerequisites.sh"
|
||||
|
||||
# Detectar OS
|
||||
if [[ -f /etc/os-release ]]; then
|
||||
. /etc/os-release
|
||||
OS_ID="$ID"
|
||||
OS_VER="$VERSION_ID"
|
||||
else
|
||||
error "No se puede detectar el sistema operativo"
|
||||
fi
|
||||
|
||||
[[ "$OS_ID" == "debian" || "$OS_ID" == "ubuntu" ]] || \
|
||||
error "Solo soportado en Debian/Ubuntu. Detectado: $OS_ID"
|
||||
|
||||
echo ""
|
||||
echo "================================================="
|
||||
echo " COCONEWS - Instalador de Prerequisites"
|
||||
echo " OS: $PRETTY_NAME"
|
||||
echo "================================================="
|
||||
echo ""
|
||||
|
||||
# =============================================================================
|
||||
# 1. PAQUETES APT BASE
|
||||
# =============================================================================
|
||||
# Comprobar espacio en disco (mínimo 10 GB libres)
|
||||
DISK_FREE_GB=$(df / --output=avail -BG | tail -1 | tr -d 'G ')
|
||||
if [[ "${DISK_FREE_GB:-0}" -lt 10 ]]; then
|
||||
warn "Espacio libre en disco: ${DISK_FREE_GB} GB (recomendado mínimo 10 GB)"
|
||||
warn "Los modelos ML necesitan ~5 GB. Continua bajo tu responsabilidad."
|
||||
fi
|
||||
|
||||
step "Actualizando repositorios apt..."
|
||||
apt-get update -qq 2>/tmp/apt-update.log || {
|
||||
echo -e "${RED}Error al actualizar repositorios apt.${NC}"
|
||||
echo " Posibles causas:"
|
||||
echo " • Sin conexión a internet"
|
||||
echo " • Repositorios con errores: cat /tmp/apt-update.log"
|
||||
echo " • Solución: apt-get update --fix-missing"
|
||||
exit 1
|
||||
}
|
||||
|
||||
step "Instalando paquetes del sistema..."
|
||||
apt-get install -y --no-install-recommends \
|
||||
curl wget git build-essential \
|
||||
ca-certificates gnupg lsb-release \
|
||||
software-properties-common apt-transport-https \
|
||||
tzdata locales \
|
||||
rsync \
|
||||
openssl \
|
||||
libpq-dev \
|
||||
libssl-dev \
|
||||
libffi-dev \
|
||||
libbz2-dev \
|
||||
libreadline-dev \
|
||||
libsqlite3-dev \
|
||||
zlib1g-dev
|
||||
info "Paquetes base instalados"
|
||||
|
||||
# =============================================================================
|
||||
# 2. POSTGRESQL 16
|
||||
# =============================================================================
|
||||
step "Instalando PostgreSQL 16..."
|
||||
if ! command -v psql &>/dev/null; then
|
||||
# Repositorio oficial de PostgreSQL
|
||||
install -d /usr/share/postgresql-common/pgdg
|
||||
curl -fsSL https://www.postgresql.org/media/keys/ACCC4CF8.asc \
|
||||
| gpg --dearmor -o /usr/share/postgresql-common/pgdg/apt.postgresql.org.gpg
|
||||
echo "deb [signed-by=/usr/share/postgresql-common/pgdg/apt.postgresql.org.gpg] \
|
||||
https://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" \
|
||||
> /etc/apt/sources.list.d/pgdg.list
|
||||
apt-get update -qq
|
||||
apt-get install -y postgresql-16 postgresql-client-16
|
||||
fi
|
||||
info "PostgreSQL: $(psql --version)"
|
||||
|
||||
# =============================================================================
|
||||
# 3. REDIS
|
||||
# =============================================================================
|
||||
step "Instalando Redis..."
|
||||
if ! command -v redis-server &>/dev/null; then
|
||||
apt-get install -y redis-server
|
||||
fi
|
||||
info "Redis: $(redis-server --version | cut -d' ' -f3)"
|
||||
|
||||
# =============================================================================
|
||||
# 4. NGINX
|
||||
# =============================================================================
|
||||
step "Instalando Nginx..."
|
||||
if ! command -v nginx &>/dev/null; then
|
||||
apt-get install -y nginx
|
||||
fi
|
||||
info "Nginx: $(nginx -v 2>&1 | cut -d'/' -f2)"
|
||||
|
||||
# =============================================================================
|
||||
# 5. PYTHON 3 + pip + venv
|
||||
# =============================================================================
|
||||
step "Instalando Python 3..."
|
||||
apt-get install -y python3 python3-pip python3-venv python3-dev
|
||||
info "Python: $(python3 --version)"
|
||||
|
||||
# =============================================================================
|
||||
# 6. NODE.JS 20 LTS
|
||||
# =============================================================================
|
||||
step "Instalando Node.js 20 LTS..."
|
||||
if ! command -v node &>/dev/null || [[ "$(node -v | tr -d 'v' | cut -d. -f1)" -lt 18 ]]; then
|
||||
curl -fsSL https://deb.nodesource.com/setup_20.x | bash -
|
||||
apt-get install -y nodejs
|
||||
fi
|
||||
info "Node.js: $(node --version) | npm: $(npm --version)"
|
||||
|
||||
# =============================================================================
|
||||
# 7. GO 1.25
|
||||
# =============================================================================
|
||||
step "Instalando Go 1.25..."
|
||||
GO_VERSION="1.25.0"
|
||||
INSTALLED_GO=$(go version 2>/dev/null | awk '{print $3}' | tr -d 'go' || echo "0")
|
||||
|
||||
# Comparar version instalada
|
||||
needs_go=false
|
||||
if ! command -v go &>/dev/null; then
|
||||
needs_go=true
|
||||
else
|
||||
IFS='.' read -ra INS <<< "$INSTALLED_GO"
|
||||
IFS='.' read -ra REQ <<< "$GO_VERSION"
|
||||
if [[ "${INS[0]}" -lt "${REQ[0]}" ]] || \
|
||||
([[ "${INS[0]}" == "${REQ[0]}" ]] && [[ "${INS[1]:-0}" -lt "${REQ[1]:-0}" ]]); then
|
||||
needs_go=true
|
||||
fi
|
||||
fi
|
||||
|
||||
if [[ "$needs_go" == "true" ]]; then
|
||||
ARCH=$(dpkg --print-architecture)
|
||||
case "$ARCH" in
|
||||
amd64) GO_ARCH="amd64" ;;
|
||||
arm64) GO_ARCH="arm64" ;;
|
||||
*) error "Arquitectura no soportada para Go: $ARCH" ;;
|
||||
esac
|
||||
step " Descargando Go ${GO_VERSION} (${GO_ARCH})..."
|
||||
curl -fsSL "https://go.dev/dl/go${GO_VERSION}.linux-${GO_ARCH}.tar.gz" -o /tmp/go.tar.gz
|
||||
rm -rf /usr/local/go
|
||||
tar -C /usr/local -xzf /tmp/go.tar.gz
|
||||
rm /tmp/go.tar.gz
|
||||
# Perfil global
|
||||
cat > /etc/profile.d/golang.sh << 'GOEOF'
|
||||
export PATH=$PATH:/usr/local/go/bin
|
||||
export GOPATH=$HOME/go
|
||||
export PATH=$PATH:$GOPATH/bin
|
||||
GOEOF
|
||||
chmod +x /etc/profile.d/golang.sh
|
||||
export PATH=$PATH:/usr/local/go/bin
|
||||
info "Go instalado: $(go version)"
|
||||
else
|
||||
export PATH=$PATH:/usr/local/go/bin
|
||||
info "Go ya instalado: $(go version)"
|
||||
fi
|
||||
|
||||
# =============================================================================
|
||||
# 8. QDRANT (binario oficial)
|
||||
# =============================================================================
|
||||
step "Instalando Qdrant..."
|
||||
QDRANT_VERSION="v1.12.1"
|
||||
QDRANT_INSTALL_DIR="/opt/rss2/qdrant"
|
||||
mkdir -p "$QDRANT_INSTALL_DIR"
|
||||
|
||||
if [[ ! -f "$QDRANT_INSTALL_DIR/qdrant" ]]; then
|
||||
ARCH=$(dpkg --print-architecture)
|
||||
case "$ARCH" in
|
||||
amd64) QDRANT_ARCH="x86_64-unknown-linux-musl" ;;
|
||||
arm64) QDRANT_ARCH="aarch64-unknown-linux-musl" ;;
|
||||
*) error "Arquitectura no soportada para Qdrant: $ARCH" ;;
|
||||
esac
|
||||
step " Descargando Qdrant ${QDRANT_VERSION}..."
|
||||
curl -fsSL \
|
||||
"https://github.com/qdrant/qdrant/releases/download/${QDRANT_VERSION}/qdrant-${QDRANT_ARCH}.tar.gz" \
|
||||
-o /tmp/qdrant.tar.gz
|
||||
tar -C "$QDRANT_INSTALL_DIR" -xzf /tmp/qdrant.tar.gz
|
||||
chmod +x "$QDRANT_INSTALL_DIR/qdrant"
|
||||
rm /tmp/qdrant.tar.gz
|
||||
info "Qdrant ${QDRANT_VERSION} instalado en ${QDRANT_INSTALL_DIR}"
|
||||
else
|
||||
info "Qdrant ya instalado en ${QDRANT_INSTALL_DIR}"
|
||||
fi
|
||||
|
||||
# =============================================================================
|
||||
# 9. USUARIO DEL SISTEMA rss2
|
||||
# =============================================================================
|
||||
step "Creando usuario del sistema 'rss2'..."
|
||||
if ! id rss2 &>/dev/null; then
|
||||
useradd -r -m -d /opt/rss2 -s /bin/bash rss2
|
||||
info "Usuario 'rss2' creado"
|
||||
else
|
||||
info "Usuario 'rss2' ya existe"
|
||||
fi
|
||||
|
||||
# Crear estructura de directorios
|
||||
mkdir -p \
|
||||
/opt/rss2/bin \
|
||||
/opt/rss2/src \
|
||||
/opt/rss2/data/wiki_images \
|
||||
/opt/rss2/data/qdrant_storage \
|
||||
/opt/rss2/hf_cache \
|
||||
/opt/rss2/models \
|
||||
/opt/rss2/frontend/dist \
|
||||
/opt/rss2/logs \
|
||||
/opt/rss2/backups
|
||||
chown -R rss2:rss2 /opt/rss2
|
||||
info "Directorios /opt/rss2 creados"
|
||||
|
||||
# =============================================================================
|
||||
# 10. PYTHON VIRTUALENV + DEPENDENCIAS BASE
|
||||
# =============================================================================
|
||||
step "Creando virtualenv Python en /opt/rss2/venv..."
|
||||
if [[ ! -d /opt/rss2/venv ]]; then
|
||||
python3 -m venv /opt/rss2/venv
|
||||
fi
|
||||
/opt/rss2/venv/bin/pip install --upgrade pip setuptools wheel -q
|
||||
info "Virtualenv listo"
|
||||
|
||||
# Dependencias base (sin los modelos pesados de ML)
|
||||
step "Instalando dependencias Python base..."
|
||||
/opt/rss2/venv/bin/pip install -q \
|
||||
psycopg2-binary \
|
||||
langdetect \
|
||||
python-dotenv \
|
||||
requests \
|
||||
beautifulsoup4 \
|
||||
lxml \
|
||||
redis \
|
||||
qdrant-client \
|
||||
numpy \
|
||||
scikit-learn \
|
||||
tqdm
|
||||
info "Dependencias Python base instaladas"
|
||||
|
||||
# =============================================================================
|
||||
# 11. DEPENDENCIAS ML (pesadas - opcional en este paso)
|
||||
# =============================================================================
|
||||
echo ""
|
||||
echo -e "${YELLOW}[?]${NC} Instalar dependencias ML pesadas ahora?"
|
||||
echo " (ctranslate2, transformers, sentence-transformers, spaCy)"
|
||||
echo " Puede tardar 20-40 minutos y usar ~5 GB de disco."
|
||||
echo -n " [s/N]: "
|
||||
read -r install_ml
|
||||
|
||||
if [[ "${install_ml,,}" == "s" || "${install_ml,,}" == "si" || "${install_ml,,}" == "y" ]]; then
|
||||
step "Instalando dependencias ML (esto tarda)..."
|
||||
/opt/rss2/venv/bin/pip install -q \
|
||||
ctranslate2>=4.0.0 \
|
||||
transformers==4.43.3 \
|
||||
sentencepiece \
|
||||
sacremoses \
|
||||
accelerate \
|
||||
sentence-transformers==3.0.1 \
|
||||
"spacy>=3.7,<4.0" \
|
||||
torch --index-url https://download.pytorch.org/whl/cpu
|
||||
info "Dependencias ML instaladas"
|
||||
|
||||
step "Descargando modelo spaCy en español..."
|
||||
/opt/rss2/venv/bin/python -m spacy download es_core_news_lg
|
||||
info "Modelo spaCy es_core_news_lg listo"
|
||||
|
||||
step "Convirtiendo modelo NLLB-200 a CTranslate2..."
|
||||
warn "Esto puede tardar 10-30 minutos y requiere ~2 GB de RAM"
|
||||
mkdir -p /opt/rss2/models
|
||||
/opt/rss2/venv/bin/python - <<'EOF'
|
||||
import os, sys
|
||||
os.makedirs("/opt/rss2/models/nllb-ct2", exist_ok=True)
|
||||
os.environ["HF_HOME"] = "/opt/rss2/hf_cache"
|
||||
try:
|
||||
from ctranslate2.converters import OpusMTConverter
|
||||
converter = OpusMTConverter("facebook/nllb-200-distilled-600M")
|
||||
converter.convert("/opt/rss2/models/nllb-ct2", quantization="int8", force=True)
|
||||
print("[OK] Modelo NLLB-200 convertido en /opt/rss2/models/nllb-ct2")
|
||||
except Exception as e:
|
||||
print(f"[ERROR] {e}")
|
||||
print("Convierte manualmente despues con: deploy/debian/convert_model.sh")
|
||||
sys.exit(0)
|
||||
EOF
|
||||
chown -R rss2:rss2 /opt/rss2/models /opt/rss2/hf_cache
|
||||
else
|
||||
warn "ML omitido. Ejecuta 'deploy/debian/install.sh' para instalarlas junto con el resto."
|
||||
warn "Sin ML: la traduccion y los embeddings no funcionaran."
|
||||
fi
|
||||
|
||||
chown -R rss2:rss2 /opt/rss2
|
||||
|
||||
# =============================================================================
|
||||
# RESUMEN Y SIGUIENTES PASOS
|
||||
# =============================================================================
|
||||
echo ""
|
||||
echo "================================================="
|
||||
echo -e " ${GREEN}Prerequisites instalados correctamente${NC}"
|
||||
echo "================================================="
|
||||
echo ""
|
||||
echo " Sistema: $PRETTY_NAME"
|
||||
echo " Go: $(go version 2>/dev/null | awk '{print $3}')"
|
||||
echo " Python: $(python3 --version)"
|
||||
echo " Node.js: $(node --version)"
|
||||
echo " PostgreSQL: $(psql --version | awk '{print $3}')"
|
||||
echo " Redis: $(redis-server --version | awk '{print $3}' | tr -d ',')"
|
||||
echo " Nginx: $(nginx -v 2>&1 | cut -d'/' -f2)"
|
||||
echo ""
|
||||
echo " Siguiente paso:"
|
||||
echo " sudo bash deploy/debian/install.sh"
|
||||
echo ""
|
||||
echo " Si algo falló durante la instalación:"
|
||||
echo " • apt falla: apt-get update --fix-missing"
|
||||
echo " • Go no descarga: verifica conectividad → curl https://go.dev"
|
||||
echo " • Qdrant no baja: descárgalo manualmente en"
|
||||
echo " https://github.com/qdrant/qdrant/releases"
|
||||
echo " • pip falla: python3 -m venv /opt/rss2/venv --clear"
|
||||
echo " • Diagnóstico: bash deploy/debian/check.sh"
|
||||
echo ""
|
||||
28
deploy/debian/systemd/rss2-backend.service
Normal file
28
deploy/debian/systemd/rss2-backend.service
Normal file
|
|
@ -0,0 +1,28 @@
|
|||
[Unit]
|
||||
Description=RSS2 Backend API (Go)
|
||||
After=network.target postgresql.service redis.service
|
||||
Requires=postgresql.service redis.service
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=rss2
|
||||
Group=rss2
|
||||
WorkingDirectory=/opt/rss2
|
||||
EnvironmentFile=/opt/rss2/.env
|
||||
# Sobreescribir hostnames Docker por localhost (los workers Python van directo a DB)
|
||||
Environment=TRANSLATION_URL=http://127.0.0.1:7790
|
||||
Environment=OLLAMA_URL=http://127.0.0.1:11434
|
||||
Environment=SPACY_URL=http://127.0.0.1:8000
|
||||
ExecStart=/opt/rss2/bin/server
|
||||
Restart=always
|
||||
RestartSec=5
|
||||
StandardOutput=journal
|
||||
StandardError=journal
|
||||
SyslogIdentifier=rss2-backend
|
||||
|
||||
# Limites de recursos
|
||||
LimitNOFILE=65536
|
||||
MemoryMax=1G
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
25
deploy/debian/systemd/rss2-categorizer.service
Normal file
25
deploy/debian/systemd/rss2-categorizer.service
Normal file
|
|
@ -0,0 +1,25 @@
|
|||
[Unit]
|
||||
Description=RSS2 Categorizer Worker (Python)
|
||||
After=network.target postgresql.service
|
||||
Requires=postgresql.service
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=rss2
|
||||
Group=rss2
|
||||
WorkingDirectory=/opt/rss2/src
|
||||
EnvironmentFile=/opt/rss2/.env
|
||||
Environment=CATEGORIZER_BATCH_SIZE=10
|
||||
Environment=CATEGORIZER_SLEEP_IDLE=5
|
||||
ExecStart=/opt/rss2/venv/bin/python -m workers.simple_categorizer_worker
|
||||
Restart=always
|
||||
RestartSec=10
|
||||
StandardOutput=journal
|
||||
StandardError=journal
|
||||
SyslogIdentifier=rss2-categorizer
|
||||
|
||||
MemoryMax=1G
|
||||
CPUQuota=200%
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
26
deploy/debian/systemd/rss2-cluster.service
Normal file
26
deploy/debian/systemd/rss2-cluster.service
Normal file
|
|
@ -0,0 +1,26 @@
|
|||
[Unit]
|
||||
Description=RSS2 Cluster Worker - Agrupacion de noticias (Python)
|
||||
After=network.target postgresql.service
|
||||
Requires=postgresql.service
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=rss2
|
||||
Group=rss2
|
||||
WorkingDirectory=/opt/rss2/src
|
||||
EnvironmentFile=/opt/rss2/.env
|
||||
Environment=EVENT_DIST_THRESHOLD=0.35
|
||||
Environment=EMB_MODEL=sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2
|
||||
Environment=HF_HOME=/opt/rss2/hf_cache
|
||||
ExecStart=/opt/rss2/venv/bin/python -m workers.cluster_worker
|
||||
Restart=always
|
||||
RestartSec=10
|
||||
StandardOutput=journal
|
||||
StandardError=journal
|
||||
SyslogIdentifier=rss2-cluster
|
||||
|
||||
MemoryMax=2G
|
||||
CPUQuota=200%
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
26
deploy/debian/systemd/rss2-discovery.service
Normal file
26
deploy/debian/systemd/rss2-discovery.service
Normal file
|
|
@ -0,0 +1,26 @@
|
|||
[Unit]
|
||||
Description=RSS2 Discovery de Feeds (Go)
|
||||
After=network.target postgresql.service
|
||||
Requires=postgresql.service
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=rss2
|
||||
Group=rss2
|
||||
WorkingDirectory=/opt/rss2
|
||||
EnvironmentFile=/opt/rss2/.env
|
||||
Environment=DISCOVERY_INTERVAL=900
|
||||
Environment=DISCOVERY_BATCH=10
|
||||
Environment=MAX_FEEDS_PER_URL=5
|
||||
ExecStart=/opt/rss2/bin/discovery
|
||||
Restart=always
|
||||
RestartSec=10
|
||||
StandardOutput=journal
|
||||
StandardError=journal
|
||||
SyslogIdentifier=rss2-discovery
|
||||
|
||||
MemoryMax=512M
|
||||
CPUQuota=100%
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
30
deploy/debian/systemd/rss2-embeddings.service
Normal file
30
deploy/debian/systemd/rss2-embeddings.service
Normal file
|
|
@ -0,0 +1,30 @@
|
|||
[Unit]
|
||||
Description=RSS2 Embeddings Worker (Python)
|
||||
After=network.target postgresql.service
|
||||
Requires=postgresql.service
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=rss2
|
||||
Group=rss2
|
||||
WorkingDirectory=/opt/rss2/src
|
||||
EnvironmentFile=/opt/rss2/.env
|
||||
Environment=EMB_MODEL=sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2
|
||||
Environment=EMB_BATCH=64
|
||||
Environment=EMB_SLEEP_IDLE=5
|
||||
Environment=EMB_LANGS=es
|
||||
Environment=EMB_LIMIT=1000
|
||||
Environment=DEVICE=cpu
|
||||
Environment=HF_HOME=/opt/rss2/hf_cache
|
||||
ExecStart=/opt/rss2/venv/bin/python -m workers.embeddings_worker
|
||||
Restart=always
|
||||
RestartSec=10
|
||||
StandardOutput=journal
|
||||
StandardError=journal
|
||||
SyslogIdentifier=rss2-embeddings
|
||||
|
||||
MemoryMax=3G
|
||||
CPUQuota=200%
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
26
deploy/debian/systemd/rss2-ingestor.service
Normal file
26
deploy/debian/systemd/rss2-ingestor.service
Normal file
|
|
@ -0,0 +1,26 @@
|
|||
[Unit]
|
||||
Description=RSS2 Ingestor RSS (Go)
|
||||
After=network.target postgresql.service
|
||||
Requires=postgresql.service
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=rss2
|
||||
Group=rss2
|
||||
WorkingDirectory=/opt/rss2
|
||||
EnvironmentFile=/opt/rss2/.env
|
||||
Environment=RSS_MAX_WORKERS=100
|
||||
Environment=RSS_POKE_INTERVAL_MIN=15
|
||||
ExecStart=/opt/rss2/bin/ingestor
|
||||
Restart=always
|
||||
RestartSec=10
|
||||
StandardOutput=journal
|
||||
StandardError=journal
|
||||
SyslogIdentifier=rss2-ingestor
|
||||
|
||||
LimitNOFILE=65536
|
||||
MemoryMax=2G
|
||||
CPUQuota=200%
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
25
deploy/debian/systemd/rss2-langdetect.service
Normal file
25
deploy/debian/systemd/rss2-langdetect.service
Normal file
|
|
@ -0,0 +1,25 @@
|
|||
[Unit]
|
||||
Description=RSS2 Language Detection Worker (Python)
|
||||
After=network.target postgresql.service
|
||||
Requires=postgresql.service
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=rss2
|
||||
Group=rss2
|
||||
WorkingDirectory=/opt/rss2/src
|
||||
EnvironmentFile=/opt/rss2/.env
|
||||
Environment=LANG_DETECT_SLEEP=60
|
||||
Environment=LANG_DETECT_BATCH=1000
|
||||
ExecStart=/opt/rss2/venv/bin/python -m workers.langdetect_worker
|
||||
Restart=always
|
||||
RestartSec=10
|
||||
StandardOutput=journal
|
||||
StandardError=journal
|
||||
SyslogIdentifier=rss2-langdetect
|
||||
|
||||
MemoryMax=512M
|
||||
CPUQuota=50%
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
26
deploy/debian/systemd/rss2-ner.service
Normal file
26
deploy/debian/systemd/rss2-ner.service
Normal file
|
|
@ -0,0 +1,26 @@
|
|||
[Unit]
|
||||
Description=RSS2 NER Worker - Extraccion de Entidades (Python)
|
||||
After=network.target postgresql.service
|
||||
Requires=postgresql.service
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=rss2
|
||||
Group=rss2
|
||||
WorkingDirectory=/opt/rss2/src
|
||||
EnvironmentFile=/opt/rss2/.env
|
||||
Environment=NER_LANG=es
|
||||
Environment=NER_BATCH=64
|
||||
Environment=HF_HOME=/opt/rss2/hf_cache
|
||||
ExecStart=/opt/rss2/venv/bin/python -m workers.ner_worker
|
||||
Restart=always
|
||||
RestartSec=10
|
||||
StandardOutput=journal
|
||||
StandardError=journal
|
||||
SyslogIdentifier=rss2-ner
|
||||
|
||||
MemoryMax=2G
|
||||
CPUQuota=200%
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
28
deploy/debian/systemd/rss2-qdrant-worker.service
Normal file
28
deploy/debian/systemd/rss2-qdrant-worker.service
Normal file
|
|
@ -0,0 +1,28 @@
|
|||
[Unit]
|
||||
Description=RSS2 Qdrant Sync Worker (Go)
|
||||
After=network.target postgresql.service rss2-qdrant.service
|
||||
Requires=postgresql.service rss2-qdrant.service
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=rss2
|
||||
Group=rss2
|
||||
WorkingDirectory=/opt/rss2
|
||||
EnvironmentFile=/opt/rss2/.env
|
||||
Environment=QDRANT_HOST=127.0.0.1
|
||||
Environment=QDRANT_PORT=6333
|
||||
Environment=QDRANT_COLLECTION=news_vectors
|
||||
Environment=QDRANT_SLEEP=30
|
||||
Environment=QDRANT_BATCH=100
|
||||
ExecStart=/opt/rss2/bin/qdrant_worker
|
||||
Restart=always
|
||||
RestartSec=10
|
||||
StandardOutput=journal
|
||||
StandardError=journal
|
||||
SyslogIdentifier=rss2-qdrant-worker
|
||||
|
||||
MemoryMax=1G
|
||||
CPUQuota=100%
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
25
deploy/debian/systemd/rss2-qdrant.service
Normal file
25
deploy/debian/systemd/rss2-qdrant.service
Normal file
|
|
@ -0,0 +1,25 @@
|
|||
[Unit]
|
||||
Description=Qdrant Vector Database
|
||||
After=network.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=rss2
|
||||
Group=rss2
|
||||
WorkingDirectory=/opt/rss2/qdrant
|
||||
ExecStart=/opt/rss2/qdrant/qdrant
|
||||
Restart=always
|
||||
RestartSec=5
|
||||
StandardOutput=journal
|
||||
StandardError=journal
|
||||
SyslogIdentifier=rss2-qdrant
|
||||
|
||||
Environment=QDRANT__SERVICE__HTTP_PORT=6333
|
||||
Environment=QDRANT__SERVICE__GRPC_PORT=6334
|
||||
Environment=QDRANT__STORAGE__STORAGE_PATH=/opt/rss2/data/qdrant_storage
|
||||
|
||||
MemoryMax=4G
|
||||
CPUQuota=400%
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
26
deploy/debian/systemd/rss2-related.service
Normal file
26
deploy/debian/systemd/rss2-related.service
Normal file
|
|
@ -0,0 +1,26 @@
|
|||
[Unit]
|
||||
Description=RSS2 Related News Worker (Go)
|
||||
After=network.target postgresql.service
|
||||
Requires=postgresql.service
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=rss2
|
||||
Group=rss2
|
||||
WorkingDirectory=/opt/rss2
|
||||
EnvironmentFile=/opt/rss2/.env
|
||||
Environment=RELATED_SLEEP=10
|
||||
Environment=RELATED_BATCH=200
|
||||
Environment=RELATED_TOPK=10
|
||||
ExecStart=/opt/rss2/bin/related
|
||||
Restart=always
|
||||
RestartSec=10
|
||||
StandardOutput=journal
|
||||
StandardError=journal
|
||||
SyslogIdentifier=rss2-related
|
||||
|
||||
MemoryMax=1G
|
||||
CPUQuota=100%
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
25
deploy/debian/systemd/rss2-scraper.service
Normal file
25
deploy/debian/systemd/rss2-scraper.service
Normal file
|
|
@ -0,0 +1,25 @@
|
|||
[Unit]
|
||||
Description=RSS2 Scraper HTML (Go)
|
||||
After=network.target postgresql.service
|
||||
Requires=postgresql.service
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=rss2
|
||||
Group=rss2
|
||||
WorkingDirectory=/opt/rss2
|
||||
EnvironmentFile=/opt/rss2/.env
|
||||
Environment=SCRAPER_SLEEP=60
|
||||
Environment=SCRAPER_BATCH=10
|
||||
ExecStart=/opt/rss2/bin/scraper
|
||||
Restart=always
|
||||
RestartSec=10
|
||||
StandardOutput=journal
|
||||
StandardError=journal
|
||||
SyslogIdentifier=rss2-scraper
|
||||
|
||||
MemoryMax=512M
|
||||
CPUQuota=100%
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
25
deploy/debian/systemd/rss2-topics.service
Normal file
25
deploy/debian/systemd/rss2-topics.service
Normal file
|
|
@ -0,0 +1,25 @@
|
|||
[Unit]
|
||||
Description=RSS2 Topics Worker (Go)
|
||||
After=network.target postgresql.service
|
||||
Requires=postgresql.service
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=rss2
|
||||
Group=rss2
|
||||
WorkingDirectory=/opt/rss2
|
||||
EnvironmentFile=/opt/rss2/.env
|
||||
Environment=TOPICS_SLEEP=10
|
||||
Environment=TOPICS_BATCH=500
|
||||
ExecStart=/opt/rss2/bin/topics
|
||||
Restart=always
|
||||
RestartSec=10
|
||||
StandardOutput=journal
|
||||
StandardError=journal
|
||||
SyslogIdentifier=rss2-topics
|
||||
|
||||
MemoryMax=512M
|
||||
CPUQuota=100%
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
26
deploy/debian/systemd/rss2-translation-scheduler.service
Normal file
26
deploy/debian/systemd/rss2-translation-scheduler.service
Normal file
|
|
@ -0,0 +1,26 @@
|
|||
[Unit]
|
||||
Description=RSS2 Translation Scheduler (Python)
|
||||
After=network.target postgresql.service
|
||||
Requires=postgresql.service
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=rss2
|
||||
Group=rss2
|
||||
WorkingDirectory=/opt/rss2/src
|
||||
EnvironmentFile=/opt/rss2/.env
|
||||
Environment=TARGET_LANGS=es
|
||||
Environment=SCHEDULER_BATCH=1000
|
||||
Environment=SCHEDULER_SLEEP=30
|
||||
ExecStart=/opt/rss2/venv/bin/python -m workers.translation_scheduler
|
||||
Restart=always
|
||||
RestartSec=10
|
||||
StandardOutput=journal
|
||||
StandardError=journal
|
||||
SyslogIdentifier=rss2-translation-scheduler
|
||||
|
||||
MemoryMax=256M
|
||||
CPUQuota=50%
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
31
deploy/debian/systemd/rss2-translator.service
Normal file
31
deploy/debian/systemd/rss2-translator.service
Normal file
|
|
@ -0,0 +1,31 @@
|
|||
[Unit]
|
||||
Description=RSS2 Translator Worker NLLB-200 (Python)
|
||||
After=network.target postgresql.service rss2-translation-scheduler.service
|
||||
Requires=postgresql.service
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=rss2
|
||||
Group=rss2
|
||||
WorkingDirectory=/opt/rss2/src
|
||||
EnvironmentFile=/opt/rss2/.env
|
||||
Environment=TARGET_LANGS=es
|
||||
Environment=TRANSLATOR_BATCH=32
|
||||
Environment=CT2_MODEL_PATH=/opt/rss2/models/nllb-ct2
|
||||
Environment=CT2_DEVICE=cpu
|
||||
Environment=CT2_COMPUTE_TYPE=int8
|
||||
Environment=UNIVERSAL_MODEL=facebook/nllb-200-distilled-600M
|
||||
Environment=HF_HOME=/opt/rss2/hf_cache
|
||||
ExecStart=/opt/rss2/venv/bin/python -m workers.ctranslator_worker
|
||||
Restart=always
|
||||
RestartSec=15
|
||||
StandardOutput=journal
|
||||
StandardError=journal
|
||||
SyslogIdentifier=rss2-translator
|
||||
|
||||
# El modelo NLLB-200 consume bastante RAM en CPU
|
||||
MemoryMax=4G
|
||||
CPUQuota=200%
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
24
deploy/debian/systemd/rss2-wiki.service
Normal file
24
deploy/debian/systemd/rss2-wiki.service
Normal file
|
|
@ -0,0 +1,24 @@
|
|||
[Unit]
|
||||
Description=RSS2 Wiki Worker - imagenes Wikipedia (Go)
|
||||
After=network.target postgresql.service
|
||||
Requires=postgresql.service
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=rss2
|
||||
Group=rss2
|
||||
WorkingDirectory=/opt/rss2
|
||||
EnvironmentFile=/opt/rss2/.env
|
||||
Environment=WIKI_SLEEP=10
|
||||
ExecStart=/opt/rss2/bin/wiki_worker
|
||||
Restart=always
|
||||
RestartSec=10
|
||||
StandardOutput=journal
|
||||
StandardError=journal
|
||||
SyslogIdentifier=rss2-wiki
|
||||
|
||||
MemoryMax=256M
|
||||
CPUQuota=50%
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
|
|
@ -1,10 +0,0 @@
|
|||
# Override local: permite acceder a Grafana en http://localhost:3001
|
||||
# La red monitoring es internal:true y compose descarta los puertos de servicios
|
||||
# que solo están en redes internas. Añadir la red frontend (no internal) permite
|
||||
# publicar el puerto 127.0.0.1:3001.
|
||||
# Uso: docker compose -f docker-compose.yml -f docker-compose.local.yml up -d
|
||||
services:
|
||||
grafana:
|
||||
networks:
|
||||
- monitoring
|
||||
- frontend
|
||||
|
|
@ -1,664 +0,0 @@
|
|||
# ==================================================================================
|
||||
# CONFIGURACIÓN DE SECRETOS - Se cargan desde .env
|
||||
# ==================================================================================
|
||||
# Las variables POSTGRES_PASSWORD, REDIS_PASSWORD, DB_PASS, SECRET_KEY, GRAFANA_PASSWORD
|
||||
# deben estar definidas en un archivo .env antes de ejecutar docker compose up
|
||||
#
|
||||
# Para generar credenciales seguras automáticamente:
|
||||
# ./generate_secure_credentials.sh --force
|
||||
#
|
||||
# Despliegue simple:
|
||||
# docker compose up -d
|
||||
# ==================================================================================
|
||||
|
||||
services:
|
||||
db:
|
||||
image: postgres:18-alpine
|
||||
container_name: rss2_db
|
||||
shm_size: 4gb
|
||||
environment:
|
||||
POSTGRES_DB: ${POSTGRES_DB:-rss}
|
||||
POSTGRES_USER: ${POSTGRES_USER:-rss}
|
||||
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
|
||||
POSTGRES_INITDB_ARGS: "--encoding=UTF8 --locale=C.UTF-8"
|
||||
LANG: C.UTF-8
|
||||
LC_ALL: C.UTF-8
|
||||
TZ: Europe/Madrid
|
||||
PGDATA: /var/lib/postgresql/data/18/main
|
||||
volumes:
|
||||
- ./data/pgdata:/var/lib/postgresql/data
|
||||
- ./init-db:/docker-entrypoint-initdb.d:rw
|
||||
- ./docker-entrypoint-db.sh:/docker-entrypoint-db.sh:ro
|
||||
entrypoint: ["bash", "/docker-entrypoint-db.sh"]
|
||||
networks:
|
||||
backend:
|
||||
aliases:
|
||||
- db
|
||||
- rss2_db
|
||||
restart: unless-stopped
|
||||
healthcheck:
|
||||
test: [ "CMD-SHELL", "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB || exit 1" ]
|
||||
# NOTA: Si falla, verifica que POSTGRES_PASSWORD esté definido en .env
|
||||
interval: 5s
|
||||
timeout: 5s
|
||||
retries: 30
|
||||
start_period: 20s
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: '2'
|
||||
memory: 8G
|
||||
reservations:
|
||||
memory: 4G
|
||||
|
||||
redis:
|
||||
image: redis:7-alpine
|
||||
container_name: rss2_redis
|
||||
environment:
|
||||
TZ: Europe/Madrid
|
||||
# SEGURIDAD: Redis con autenticación
|
||||
command: >
|
||||
redis-server --appendonly yes --maxmemory 512mb --maxmemory-policy allkeys-lru --requirepass ${REDIS_PASSWORD:-!ERROR!REDIS_PASSWORD_NEEDED!}
|
||||
volumes:
|
||||
- ./data/redis-data:/data
|
||||
- /etc/timezone:/etc/timezone:ro
|
||||
- /etc/localtime:/etc/localtime:ro
|
||||
networks:
|
||||
backend:
|
||||
aliases:
|
||||
- redis
|
||||
- rss2_redis
|
||||
restart: unless-stopped
|
||||
healthcheck:
|
||||
# VALIDACIÓN: Si REDIS_PASSWORD está vacío, esto fallará y detendrá el despliegue
|
||||
test: [ "CMD", "redis-cli", "--no-auth-warning", "-a", "${REDIS_PASSWORD}", "ping" ]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 5
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
memory: 768M
|
||||
reservations:
|
||||
memory: 512M
|
||||
|
||||
rss-ingestor-go:
|
||||
build:
|
||||
context: ./rss-ingestor-go
|
||||
dockerfile: Dockerfile
|
||||
container_name: rss2_ingestor_go
|
||||
environment:
|
||||
DB_HOST: db
|
||||
DB_PORT: 5432
|
||||
DB_NAME: ${DB_NAME:-rss}
|
||||
DB_USER: ${DB_USER:-rss}
|
||||
DB_PASS: ${DB_PASS}
|
||||
RSS_MAX_WORKERS: ${RSS_MAX_WORKERS:-100}
|
||||
RSS_POKE_INTERVAL_MIN: ${RSS_POKE_INTERVAL_MIN:-60}
|
||||
RSS_HOST_DELAY_MS: ${RSS_HOST_DELAY_MS:-800}
|
||||
TZ: Europe/Madrid
|
||||
networks:
|
||||
- backend
|
||||
depends_on:
|
||||
db:
|
||||
condition: service_healthy
|
||||
restart: unless-stopped
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: '2'
|
||||
memory: 2G
|
||||
reservations:
|
||||
memory: 512M
|
||||
|
||||
langdetect:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile
|
||||
container_name: rss2_langdetect_py
|
||||
command: bash -lc "python -m workers.langdetect_worker"
|
||||
environment:
|
||||
DB_HOST: db
|
||||
DB_PORT: 5432
|
||||
DB_NAME: ${DB_NAME:-rss}
|
||||
DB_USER: ${DB_USER:-rss}
|
||||
DB_PASS: ${DB_PASS}
|
||||
LANG_DETECT_SLEEP: 60
|
||||
LANG_DETECT_BATCH: 1000
|
||||
TZ: Europe/Madrid
|
||||
volumes:
|
||||
- ./workers:/app/workers
|
||||
networks:
|
||||
- backend
|
||||
depends_on:
|
||||
db:
|
||||
condition: service_healthy
|
||||
restart: unless-stopped
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: '0.5'
|
||||
memory: 512M
|
||||
|
||||
# ==================================================================================
|
||||
# SCRAPER WORKER (Go) - Extrae artículos de URLs
|
||||
# ==================================================================================
|
||||
scraper:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile.scraper
|
||||
container_name: rss2_scraper
|
||||
environment:
|
||||
DB_HOST: db
|
||||
DB_PORT: 5432
|
||||
DB_NAME: ${DB_NAME:-rss}
|
||||
DB_USER: ${DB_USER:-rss}
|
||||
DB_PASS: ${DB_PASS}
|
||||
SCRAPER_SLEEP: 120
|
||||
SCRAPER_BATCH: 10
|
||||
SCRAPER_ENRICH_LIMIT: 10
|
||||
SCRAPER_WORKERS: ${SCRAPER_WORKERS:-2}
|
||||
TZ: Europe/Madrid
|
||||
networks:
|
||||
- backend
|
||||
depends_on:
|
||||
db:
|
||||
condition: service_healthy
|
||||
restart: unless-stopped
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: '1'
|
||||
memory: 512M
|
||||
|
||||
# ==================================================================================
|
||||
# DISCOVERY WORKER (Go) - Descubre RSS feeds
|
||||
# ==================================================================================
|
||||
discovery:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile.discovery
|
||||
container_name: rss2_discovery
|
||||
environment:
|
||||
DB_HOST: db
|
||||
DB_PORT: 5432
|
||||
DB_NAME: ${DB_NAME:-rss}
|
||||
DB_USER: ${DB_USER:-rss}
|
||||
DB_PASS: ${DB_PASS}
|
||||
DISCOVERY_INTERVAL: 900
|
||||
DISCOVERY_BATCH: 10
|
||||
MAX_FEEDS_PER_URL: 5
|
||||
TZ: Europe/Madrid
|
||||
networks:
|
||||
- backend
|
||||
depends_on:
|
||||
db:
|
||||
condition: service_healthy
|
||||
restart: unless-stopped
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: '1'
|
||||
memory: 512M
|
||||
|
||||
# ==================================================================================
|
||||
# WIKI WORKER (Go) - Wikipedia info and thumbnails
|
||||
# ==================================================================================
|
||||
wiki-worker:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile.wiki
|
||||
container_name: rss2_wiki_worker
|
||||
environment:
|
||||
DB_HOST: db
|
||||
DB_PORT: 5432
|
||||
DB_NAME: ${DB_NAME:-rss}
|
||||
DB_USER: ${DB_USER:-rss}
|
||||
DB_PASS: ${DB_PASS}
|
||||
WIKI_SLEEP: 10
|
||||
WIKI_BATCH: 20
|
||||
TZ: Europe/Madrid
|
||||
volumes:
|
||||
- ./data/wiki_images:/app/data/wiki_images
|
||||
networks:
|
||||
- backend
|
||||
depends_on:
|
||||
db:
|
||||
condition: service_healthy
|
||||
restart: unless-stopped
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: '0.5'
|
||||
memory: 256M
|
||||
|
||||
# ==================================================================================
|
||||
# BACKEND GO (API REST)
|
||||
# ==================================================================================
|
||||
backend-go:
|
||||
build:
|
||||
context: ./backend
|
||||
dockerfile: Dockerfile
|
||||
container_name: rss2_backend_go
|
||||
environment:
|
||||
TZ: Europe/Madrid
|
||||
DATABASE_URL: postgres://${POSTGRES_USER:-rss}:${POSTGRES_PASSWORD}@db:5432/${POSTGRES_DB:-rss}?sslmode=disable
|
||||
REDIS_URL: redis://:${REDIS_PASSWORD:-rss_redis_pass_2024}@redis:6379
|
||||
SECRET_KEY: ${SECRET_KEY:-change_this_to_a_long_random_string}
|
||||
SERVER_PORT: "8080"
|
||||
volumes:
|
||||
- ./data/wiki_images:/app/data/wiki_images
|
||||
networks:
|
||||
- backend
|
||||
- frontend
|
||||
depends_on:
|
||||
db:
|
||||
condition: service_healthy
|
||||
redis:
|
||||
condition: service_healthy
|
||||
restart: unless-stopped
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: '2'
|
||||
memory: 512M
|
||||
|
||||
# ==================================================================================
|
||||
# FRONTEND REACT
|
||||
# ==================================================================================
|
||||
rss2_frontend:
|
||||
build:
|
||||
context: ./frontend
|
||||
dockerfile: Dockerfile
|
||||
container_name: rss2_frontend
|
||||
environment:
|
||||
TZ: Europe/Madrid
|
||||
VITE_API_URL: /api
|
||||
networks:
|
||||
- frontend
|
||||
depends_on:
|
||||
- backend-go
|
||||
restart: unless-stopped
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: '1'
|
||||
memory: 512M
|
||||
|
||||
# ==================================================================================
|
||||
# NGINX (Puerto 8001 - sirve React + proxy API)
|
||||
# ==================================================================================
|
||||
nginx:
|
||||
image: nginx:alpine
|
||||
container_name: rss2_nginx
|
||||
ports:
|
||||
- "8888:80"
|
||||
volumes:
|
||||
- ./nginx.conf:/etc/nginx/nginx.conf:ro
|
||||
networks:
|
||||
- frontend
|
||||
depends_on:
|
||||
- rss2_frontend
|
||||
- backend-go
|
||||
restart: unless-stopped
|
||||
|
||||
# ==================================================================================
|
||||
# TRANSLATOR CPU (CTranslate2) - Scale with: docker compose up -d --scale translator=3
|
||||
# ==================================================================================
|
||||
translator:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile.translator
|
||||
image: rss2-translator:latest
|
||||
command: bash -lc "python -m workers.ctranslator_worker"
|
||||
environment:
|
||||
DB_HOST: db
|
||||
DB_PORT: 5432
|
||||
DB_NAME: ${DB_NAME:-rss}
|
||||
DB_USER: ${DB_USER:-rss}
|
||||
DB_PASS: ${DB_PASS}
|
||||
TARGET_LANGS: es
|
||||
TRANSLATOR_BATCH: ${TRANSLATOR_BATCH:-128}
|
||||
CT2_MODEL_PATH: /app/models/nllb-ct2
|
||||
CT2_DEVICE: cpu
|
||||
CT2_COMPUTE_TYPE: int8
|
||||
CT2_INTRA_THREADS: 4
|
||||
UNIVERSAL_MODEL: facebook/nllb-200-distilled-600M
|
||||
HF_HOME: /app/hf_cache
|
||||
TZ: Europe/Madrid
|
||||
TRANSLATOR_ID: ${TRANSLATOR_ID:-}
|
||||
REDIS_URL: redis://:${REDIS_PASSWORD:-rss_redis_pass_2024}@redis:6379
|
||||
PYTORCH_ENABLE_MPS_FALLBACK: 1
|
||||
volumes:
|
||||
- ./workers:/app/workers
|
||||
- ./hf_cache:/app/hf_cache
|
||||
- ./models:/app/models
|
||||
networks:
|
||||
- backend
|
||||
depends_on:
|
||||
db:
|
||||
condition: service_healthy
|
||||
restart: unless-stopped
|
||||
# Prioridad BAJA: el traductor no debe acaparar toda la CPU.
|
||||
# cpu_shares menor que el resto (1024) + tope que deje CPU para NER, embeddings, etc.
|
||||
cpu_shares: 512
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: '4'
|
||||
memory: 6G
|
||||
|
||||
# ==================================================================================
|
||||
# TRANSLATOR CPU Nº2 - Segundo worker CPU (paralelo al translator)
|
||||
# ==================================================================================
|
||||
translator-gpu:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile.translator
|
||||
image: rss2-translator:latest
|
||||
command: bash -lc "python -m workers.ctranslator_worker"
|
||||
environment:
|
||||
DB_HOST: db
|
||||
DB_PORT: 5432
|
||||
DB_NAME: ${DB_NAME:-rss}
|
||||
DB_USER: ${DB_USER:-rss}
|
||||
DB_PASS: ${DB_PASS}
|
||||
TARGET_LANGS: es
|
||||
TRANSLATOR_BATCH: ${TRANSLATOR_BATCH:-128}
|
||||
CT2_MODEL_PATH: /app/models/nllb-ct2
|
||||
CT2_DEVICE: cpu
|
||||
CT2_COMPUTE_TYPE: int8
|
||||
CT2_INTRA_THREADS: 4
|
||||
UNIVERSAL_MODEL: facebook/nllb-200-distilled-600M
|
||||
HF_HOME: /app/hf_cache
|
||||
TZ: Europe/Madrid
|
||||
TRANSLATOR_ID: ${TRANSLATOR_ID:-}
|
||||
REDIS_URL: redis://:${REDIS_PASSWORD:-rss_redis_pass_2024}@redis:6379
|
||||
PYTORCH_ENABLE_MPS_FALLBACK: 1
|
||||
volumes:
|
||||
- ./workers:/app/workers
|
||||
- ./hf_cache:/app/hf_cache
|
||||
- ./models:/app/models
|
||||
networks:
|
||||
- backend
|
||||
depends_on:
|
||||
db:
|
||||
condition: service_healthy
|
||||
restart: unless-stopped
|
||||
# Prioridad BAJA: mismo trato que el translator.
|
||||
cpu_shares: 512
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: '4'
|
||||
memory: 6G
|
||||
|
||||
# ==================================================================================
|
||||
# TRANSLATION SCHEDULER - Creates translation jobs
|
||||
# ==================================================================================
|
||||
translation-scheduler:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile.scheduler
|
||||
image: rss2-scheduler:latest
|
||||
container_name: rss2_translation_scheduler
|
||||
environment:
|
||||
DB_HOST: db
|
||||
DB_PORT: 5432
|
||||
DB_NAME: ${DB_NAME:-rss}
|
||||
DB_USER: ${DB_USER:-rss}
|
||||
DB_PASS: ${DB_PASS}
|
||||
TARGET_LANGS: es
|
||||
SCHEDULER_BATCH: 200
|
||||
SCHEDULER_AGE_DAYS: 30
|
||||
SCHEDULER_SLEEP: 30
|
||||
TZ: Europe/Madrid
|
||||
volumes:
|
||||
- ./workers:/app/workers
|
||||
networks:
|
||||
- backend
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: '0.5'
|
||||
memory: 256M
|
||||
depends_on:
|
||||
db:
|
||||
condition: service_healthy
|
||||
restart: unless-stopped
|
||||
|
||||
embeddings:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile
|
||||
container_name: rss2_embeddings_py
|
||||
command: bash -lc "python -m workers.embeddings_worker"
|
||||
environment:
|
||||
DB_HOST: db
|
||||
DB_PORT: 5432
|
||||
DB_NAME: ${DB_NAME:-rss}
|
||||
DB_USER: ${DB_USER:-rss}
|
||||
DB_PASS: ${DB_PASS}
|
||||
EMB_MODEL: sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2
|
||||
EMB_BATCH: 32
|
||||
EMB_SLEEP_IDLE: 5
|
||||
EMB_LANGS: es
|
||||
EMB_LIMIT: 300
|
||||
DEVICE: cpu
|
||||
HF_HOME: /app/hf_cache
|
||||
TZ: Europe/Madrid
|
||||
volumes:
|
||||
- ./workers:/app/workers
|
||||
- ./hf_cache:/app/hf_cache
|
||||
networks:
|
||||
- backend
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: '2'
|
||||
memory: 6G
|
||||
depends_on:
|
||||
db:
|
||||
condition: service_healthy
|
||||
restart: unless-stopped
|
||||
|
||||
# ==================================================================================
|
||||
# TOPICS WORKER (Go) - Matching temas y países
|
||||
# ==================================================================================
|
||||
topics:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile.topics
|
||||
container_name: rss2_topics
|
||||
environment:
|
||||
DB_HOST: db
|
||||
DB_PORT: 5432
|
||||
DB_NAME: ${DB_NAME:-rss}
|
||||
DB_USER: ${DB_USER:-rss}
|
||||
DB_PASS: ${DB_PASS}
|
||||
TOPICS_SLEEP: 10
|
||||
TOPICS_BATCH: 500
|
||||
TZ: Europe/Madrid
|
||||
networks:
|
||||
- backend
|
||||
depends_on:
|
||||
db:
|
||||
condition: service_healthy
|
||||
restart: unless-stopped
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: '1'
|
||||
memory: 512M
|
||||
|
||||
# ==================================================================================
|
||||
# RELATED WORKER (Go) - Noticias relacionadas
|
||||
# ==================================================================================
|
||||
related:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile.related
|
||||
container_name: rss2_related
|
||||
environment:
|
||||
DB_HOST: db
|
||||
DB_PORT: 5432
|
||||
DB_NAME: ${DB_NAME:-rss}
|
||||
DB_USER: ${DB_USER:-rss}
|
||||
DB_PASS: ${DB_PASS}
|
||||
RELATED_SLEEP: 10
|
||||
RELATED_BATCH: 200
|
||||
RELATED_TOPK: 10
|
||||
EMB_MODEL: sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2
|
||||
TZ: Europe/Madrid
|
||||
networks:
|
||||
- backend
|
||||
depends_on:
|
||||
db:
|
||||
condition: service_healthy
|
||||
restart: unless-stopped
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: '1'
|
||||
memory: 1G
|
||||
|
||||
qdrant:
|
||||
image: qdrant/qdrant:latest
|
||||
container_name: rss2_qdrant
|
||||
environment:
|
||||
TZ: Europe/Madrid
|
||||
QDRANT__SERVICE__GRPC_PORT: 6334
|
||||
volumes:
|
||||
- ./data/qdrant_storage:/qdrant/storage
|
||||
- /etc/timezone:/etc/timezone:ro
|
||||
- /etc/localtime:/etc/localtime:ro
|
||||
networks:
|
||||
- backend
|
||||
restart: unless-stopped
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: '4'
|
||||
memory: 4G
|
||||
reservations:
|
||||
memory: 2G
|
||||
|
||||
# ==================================================================================
|
||||
# QDRANT WORKER (Go) - Vectorización y búsqueda semántica
|
||||
# ==================================================================================
|
||||
qdrant-worker:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile.qdrant
|
||||
container_name: rss2_qdrant_worker
|
||||
environment:
|
||||
DB_HOST: db
|
||||
DB_PORT: 5432
|
||||
DB_NAME: ${DB_NAME:-rss}
|
||||
DB_USER: ${DB_USER:-rss}
|
||||
DB_PASS: ${DB_PASS}
|
||||
QDRANT_HOST: qdrant
|
||||
QDRANT_PORT: 6333
|
||||
QDRANT_COLLECTION: news_vectors
|
||||
OLLAMA_URL: http://ollama:11434
|
||||
QDRANT_SLEEP: 30
|
||||
QDRANT_BATCH: 100
|
||||
TZ: Europe/Madrid
|
||||
networks:
|
||||
- backend
|
||||
depends_on:
|
||||
db:
|
||||
condition: service_healthy
|
||||
qdrant:
|
||||
condition: service_started
|
||||
restart: unless-stopped
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: '1'
|
||||
memory: 1G
|
||||
|
||||
# ==================================================================================
|
||||
# NER WORKER (Python) - Extracción de entidades
|
||||
# ==================================================================================
|
||||
ner:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile
|
||||
container_name: rss2_ner
|
||||
command: bash -lc "python -m workers.ner_worker"
|
||||
environment:
|
||||
DB_HOST: db
|
||||
DB_PORT: 5432
|
||||
DB_NAME: ${DB_NAME:-rss}
|
||||
DB_USER: ${DB_USER:-rss}
|
||||
DB_PASS: ${DB_PASS}
|
||||
NER_LANG: es
|
||||
NER_BATCH: 24
|
||||
HF_HOME: /app/hf_cache
|
||||
TZ: Europe/Madrid
|
||||
volumes:
|
||||
- ./workers:/app/workers
|
||||
- ./hf_cache:/app/hf_cache
|
||||
networks:
|
||||
- backend
|
||||
depends_on:
|
||||
db:
|
||||
condition: service_healthy
|
||||
restart: unless-stopped
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: '2'
|
||||
memory: 2G
|
||||
|
||||
# ==================================================================================
|
||||
# CLUSTER WORKER (Python) - Agrupación de noticias
|
||||
# ==================================================================================
|
||||
cluster:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile
|
||||
container_name: rss2_cluster_py
|
||||
command: bash -lc "python -m workers.cluster_worker"
|
||||
environment:
|
||||
DB_HOST: db
|
||||
DB_PORT: 5432
|
||||
DB_NAME: ${DB_NAME:-rss}
|
||||
DB_USER: ${DB_USER:-rss}
|
||||
DB_PASS: ${DB_PASS}
|
||||
EVENT_DIST_THRESHOLD: 0.35
|
||||
EMB_MODEL: sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2
|
||||
TZ: Europe/Madrid
|
||||
volumes:
|
||||
- ./workers:/app/workers
|
||||
networks:
|
||||
- backend
|
||||
depends_on:
|
||||
db:
|
||||
condition: service_healthy
|
||||
restart: unless-stopped
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: '2'
|
||||
memory: 2G
|
||||
|
||||
# ==================================================================================
|
||||
# REDES SEGMENTADAS
|
||||
# ==================================================================================
|
||||
networks:
|
||||
# Red frontal - Solo nginx y web app
|
||||
frontend:
|
||||
name: rss2_frontend
|
||||
driver: bridge
|
||||
internal: false
|
||||
|
||||
# Red backend - Base de datos, workers, redis, qdrant
|
||||
backend:
|
||||
name: rss2_backend
|
||||
driver: bridge
|
||||
internal: false # Acceso externo permitido (necesario para ingestor)
|
||||
|
||||
volumes:
|
||||
torch_extensions:
|
||||
|
|
@ -1,802 +0,0 @@
|
|||
# ==================================================================================
|
||||
# CONFIGURACIÓN DE SECRETOS - Se cargan desde .env
|
||||
# ==================================================================================
|
||||
# Las variables POSTGRES_PASSWORD, REDIS_PASSWORD, DB_PASS, SECRET_KEY, GRAFANA_PASSWORD
|
||||
# deben estar definidas en un archivo .env antes de ejecutar docker compose up
|
||||
#
|
||||
# Para generar credenciales seguras automáticamente:
|
||||
# ./generate_secure_credentials.sh --force
|
||||
#
|
||||
# Despliegue simple:
|
||||
# docker compose up -d
|
||||
# ==================================================================================
|
||||
|
||||
services:
|
||||
db:
|
||||
image: postgres:18-alpine
|
||||
container_name: rss2_db
|
||||
shm_size: 4gb
|
||||
environment:
|
||||
POSTGRES_DB: ${POSTGRES_DB:-rss}
|
||||
POSTGRES_USER: ${POSTGRES_USER:-rss}
|
||||
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
|
||||
POSTGRES_INITDB_ARGS: "--encoding=UTF8 --locale=C.UTF-8"
|
||||
LANG: C.UTF-8
|
||||
LC_ALL: C.UTF-8
|
||||
TZ: Europe/Madrid
|
||||
PGDATA: /var/lib/postgresql/data/18/main
|
||||
volumes:
|
||||
- ./data/pgdata:/var/lib/postgresql/data
|
||||
- ./init-db:/docker-entrypoint-initdb.d:rw
|
||||
- ./docker-entrypoint-db.sh:/docker-entrypoint-db.sh:ro
|
||||
entrypoint: ["bash", "/docker-entrypoint-db.sh"]
|
||||
networks:
|
||||
backend:
|
||||
aliases:
|
||||
- db
|
||||
- rss2_db
|
||||
restart: unless-stopped
|
||||
healthcheck:
|
||||
test: [ "CMD-SHELL", "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB || exit 1" ]
|
||||
# NOTA: Si falla, verifica que POSTGRES_PASSWORD esté definido en .env
|
||||
interval: 5s
|
||||
timeout: 5s
|
||||
retries: 30
|
||||
start_period: 20s
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
memory: 8G
|
||||
reservations:
|
||||
memory: 4G
|
||||
|
||||
redis:
|
||||
image: redis:7-alpine
|
||||
container_name: rss2_redis
|
||||
environment:
|
||||
TZ: Europe/Madrid
|
||||
# SEGURIDAD: Redis con autenticación
|
||||
command: >
|
||||
redis-server --appendonly yes --maxmemory 512mb --maxmemory-policy allkeys-lru --requirepass ${REDIS_PASSWORD:-!ERROR!REDIS_PASSWORD_NEEDED!}
|
||||
volumes:
|
||||
- ./data/redis-data:/data
|
||||
- /etc/timezone:/etc/timezone:ro
|
||||
- /etc/localtime:/etc/localtime:ro
|
||||
networks:
|
||||
backend:
|
||||
aliases:
|
||||
- redis
|
||||
- rss2_redis
|
||||
restart: unless-stopped
|
||||
healthcheck:
|
||||
# VALIDACIÓN: Si REDIS_PASSWORD está vacío, esto fallará y detendrá el despliegue
|
||||
test: [ "CMD", "redis-cli", "--no-auth-warning", "-a", "${REDIS_PASSWORD}", "ping" ]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 5
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
memory: 768M
|
||||
reservations:
|
||||
memory: 512M
|
||||
|
||||
rss-ingestor-go:
|
||||
build:
|
||||
context: ./rss-ingestor-go
|
||||
dockerfile: Dockerfile
|
||||
container_name: rss2_ingestor_go
|
||||
environment:
|
||||
DB_HOST: db
|
||||
DB_PORT: 5432
|
||||
DB_NAME: ${DB_NAME:-rss}
|
||||
DB_USER: ${DB_USER:-rss}
|
||||
DB_PASS: ${DB_PASS}
|
||||
RSS_MAX_WORKERS: 100
|
||||
RSS_POKE_INTERVAL_MIN: 60
|
||||
TZ: Europe/Madrid
|
||||
networks:
|
||||
- backend
|
||||
depends_on:
|
||||
db:
|
||||
condition: service_healthy
|
||||
restart: unless-stopped
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: '2'
|
||||
memory: 2G
|
||||
reservations:
|
||||
memory: 512M
|
||||
|
||||
langdetect:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile
|
||||
container_name: rss2_langdetect_py
|
||||
command: bash -lc "python -m workers.langdetect_worker"
|
||||
environment:
|
||||
DB_HOST: db
|
||||
DB_PORT: 5432
|
||||
DB_NAME: ${DB_NAME:-rss}
|
||||
DB_USER: ${DB_USER:-rss}
|
||||
DB_PASS: ${DB_PASS}
|
||||
LANG_DETECT_SLEEP: 60
|
||||
LANG_DETECT_BATCH: 1000
|
||||
TZ: Europe/Madrid
|
||||
volumes:
|
||||
- ./workers:/app/workers
|
||||
networks:
|
||||
- backend
|
||||
depends_on:
|
||||
db:
|
||||
condition: service_healthy
|
||||
restart: unless-stopped
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: '0.5'
|
||||
memory: 512M
|
||||
|
||||
# ==================================================================================
|
||||
# SCRAPER WORKER (Go) - Extrae artículos de URLs
|
||||
# ==================================================================================
|
||||
scraper:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile.scraper
|
||||
container_name: rss2_scraper
|
||||
environment:
|
||||
DB_HOST: db
|
||||
DB_PORT: 5432
|
||||
DB_NAME: ${DB_NAME:-rss}
|
||||
DB_USER: ${DB_USER:-rss}
|
||||
DB_PASS: ${DB_PASS}
|
||||
SCRAPER_SLEEP: 60
|
||||
SCRAPER_BATCH: 10
|
||||
SCRAPER_ENRICH_LIMIT: 20
|
||||
TZ: Europe/Madrid
|
||||
networks:
|
||||
- backend
|
||||
depends_on:
|
||||
db:
|
||||
condition: service_healthy
|
||||
restart: unless-stopped
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: '1'
|
||||
memory: 512M
|
||||
|
||||
# ==================================================================================
|
||||
# DISCOVERY WORKER (Go) - Descubre RSS feeds
|
||||
# ==================================================================================
|
||||
discovery:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile.discovery
|
||||
container_name: rss2_discovery
|
||||
environment:
|
||||
DB_HOST: db
|
||||
DB_PORT: 5432
|
||||
DB_NAME: ${DB_NAME:-rss}
|
||||
DB_USER: ${DB_USER:-rss}
|
||||
DB_PASS: ${DB_PASS}
|
||||
DISCOVERY_INTERVAL: 900
|
||||
DISCOVERY_BATCH: 10
|
||||
MAX_FEEDS_PER_URL: 5
|
||||
TZ: Europe/Madrid
|
||||
networks:
|
||||
- backend
|
||||
depends_on:
|
||||
db:
|
||||
condition: service_healthy
|
||||
restart: unless-stopped
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: '1'
|
||||
memory: 512M
|
||||
|
||||
# ==================================================================================
|
||||
# WIKI WORKER (Go) - Wikipedia info and thumbnails
|
||||
# ==================================================================================
|
||||
wiki-worker:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile.wiki
|
||||
container_name: rss2_wiki_worker
|
||||
environment:
|
||||
DB_HOST: db
|
||||
DB_PORT: 5432
|
||||
DB_NAME: ${DB_NAME:-rss}
|
||||
DB_USER: ${DB_USER:-rss}
|
||||
DB_PASS: ${DB_PASS}
|
||||
WIKI_SLEEP: 10
|
||||
TZ: Europe/Madrid
|
||||
volumes:
|
||||
- ./data/wiki_images:/app/data/wiki_images
|
||||
networks:
|
||||
- backend
|
||||
depends_on:
|
||||
db:
|
||||
condition: service_healthy
|
||||
restart: unless-stopped
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: '0.5'
|
||||
memory: 256M
|
||||
|
||||
# ==================================================================================
|
||||
# BACKEND GO (API REST)
|
||||
# ==================================================================================
|
||||
backend-go:
|
||||
build:
|
||||
context: ./backend
|
||||
dockerfile: Dockerfile
|
||||
container_name: rss2_backend_go
|
||||
environment:
|
||||
TZ: Europe/Madrid
|
||||
DATABASE_URL: postgres://${POSTGRES_USER:-rss}:${POSTGRES_PASSWORD}@db:5432/${POSTGRES_DB:-rss}?sslmode=disable
|
||||
REDIS_URL: redis://:${REDIS_PASSWORD:-rss_redis_pass_2024}@redis:6379
|
||||
SECRET_KEY: ${SECRET_KEY:-change_this_to_a_long_random_string}
|
||||
SERVER_PORT: "8080"
|
||||
volumes:
|
||||
- ./data/wiki_images:/app/data/wiki_images
|
||||
networks:
|
||||
- backend
|
||||
- frontend
|
||||
depends_on:
|
||||
db:
|
||||
condition: service_healthy
|
||||
redis:
|
||||
condition: service_healthy
|
||||
restart: unless-stopped
|
||||
|
||||
# ==================================================================================
|
||||
# FRONTEND REACT
|
||||
# ==================================================================================
|
||||
rss2_frontend:
|
||||
build:
|
||||
context: ./frontend
|
||||
dockerfile: Dockerfile
|
||||
container_name: rss2_frontend
|
||||
environment:
|
||||
TZ: Europe/Madrid
|
||||
VITE_API_URL: /api
|
||||
networks:
|
||||
- frontend
|
||||
depends_on:
|
||||
- backend-go
|
||||
restart: unless-stopped
|
||||
|
||||
# ==================================================================================
|
||||
# NGINX (Puerto 8001 - sirve React + proxy API)
|
||||
# ==================================================================================
|
||||
nginx:
|
||||
image: nginx:alpine
|
||||
container_name: rss2_nginx
|
||||
ports:
|
||||
- "8888:80"
|
||||
volumes:
|
||||
- ./nginx.conf:/etc/nginx/nginx.conf:ro
|
||||
networks:
|
||||
- frontend
|
||||
depends_on:
|
||||
- rss2_frontend
|
||||
- backend-go
|
||||
restart: unless-stopped
|
||||
|
||||
# ==================================================================================
|
||||
# TRANSLATOR CPU (CTranslate2) - Scale with: docker compose up -d --scale translator=3
|
||||
# ==================================================================================
|
||||
translator:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile.translator
|
||||
image: rss2-translator:latest
|
||||
command: bash -lc "python -m workers.ctranslator_worker"
|
||||
environment:
|
||||
DB_HOST: db
|
||||
DB_PORT: 5432
|
||||
DB_NAME: ${DB_NAME:-rss}
|
||||
DB_USER: ${DB_USER:-rss}
|
||||
DB_PASS: ${DB_PASS}
|
||||
TARGET_LANGS: es
|
||||
TRANSLATOR_BATCH: 32
|
||||
CT2_MODEL_PATH: /app/models/nllb-ct2
|
||||
CT2_DEVICE: cpu
|
||||
CT2_COMPUTE_TYPE: int8
|
||||
UNIVERSAL_MODEL: facebook/nllb-200-distilled-600M
|
||||
HF_HOME: /app/hf_cache
|
||||
TZ: Europe/Madrid
|
||||
TRANSLATOR_ID: ${TRANSLATOR_ID:-}
|
||||
PYTORCH_ENABLE_MPS_FALLBACK: 1
|
||||
volumes:
|
||||
- ./workers:/app/workers
|
||||
- ./hf_cache:/app/hf_cache
|
||||
- ./models:/app/models
|
||||
networks:
|
||||
- backend
|
||||
profiles:
|
||||
- cpu-only
|
||||
depends_on:
|
||||
db:
|
||||
condition: service_healthy
|
||||
restart: unless-stopped
|
||||
|
||||
# ==================================================================================
|
||||
# TRANSLATOR GPU - Multiple instances with --scale for parallel GPU processing
|
||||
# Configure multiple GPU workers: docker compose up -d --scale translator-gpu=2
|
||||
# ==================================================================================
|
||||
translator-gpu:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile.translator-gpu
|
||||
image: rss2-translator-gpu:latest
|
||||
command: bash -lc "python -m workers.ctranslator_worker"
|
||||
environment:
|
||||
DB_HOST: db
|
||||
DB_PORT: 5432
|
||||
DB_NAME: ${DB_NAME:-rss}
|
||||
DB_USER: ${DB_USER:-rss}
|
||||
DB_PASS: ${DB_PASS}
|
||||
TARGET_LANGS: es
|
||||
TRANSLATOR_BATCH: 128
|
||||
CT2_MODEL_PATH: /app/models/nllb-ct2
|
||||
CT2_DEVICE: cuda
|
||||
CT2_COMPUTE_TYPE: float16
|
||||
UNIVERSAL_MODEL: facebook/nllb-200-distilled-600M
|
||||
HF_HOME: /app/hf_cache
|
||||
TZ: Europe/Madrid
|
||||
TRANSLATOR_ID: ${TRANSLATOR_ID:-}
|
||||
PYTORCH_CUDA_ALLOC_CONF: max_split_size_mb:512
|
||||
NCCL_DEBUG: INFO
|
||||
volumes:
|
||||
- ./workers:/app/workers
|
||||
- ./hf_cache:/app/hf_cache
|
||||
- ./models:/app/models
|
||||
- /dev/nvidia:/dev/nvidia
|
||||
- /usr/local/nvidia:/usr/local/nvidia:ro
|
||||
networks:
|
||||
- backend
|
||||
profiles:
|
||||
- gpu
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: 1
|
||||
capabilities: [gpu]
|
||||
depends_on:
|
||||
db:
|
||||
condition: service_healthy
|
||||
restart: unless-stopped
|
||||
|
||||
# ==================================================================================
|
||||
# REMOTE TRANSLATOR WORKER - Worker remoto que se conecta por WebSocket
|
||||
# ==================================================================================
|
||||
remote-translator:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile.remote-worker
|
||||
container_name: rss2_remote_translator
|
||||
environment:
|
||||
WORKER_NAME: ${WORKER_NAME:-remote-worker-1}
|
||||
WORKER_API_KEY: ${WORKER_API_KEY:-}
|
||||
WORKER_SERVER: ${WORKER_SERVER:-ws://backend-go:8080/ws/worker}
|
||||
CT2_DEVICE: ${CT2_DEVICE:-cuda}
|
||||
CT2_MODEL_PATH: /app/models/nllb-ct2
|
||||
CT2_COMPUTE_TYPE: ${CT2_COMPUTE_TYPE:-float16}
|
||||
UNIVERSAL_MODEL: facebook/nllb-200-1.3B
|
||||
HF_HOME: /app/hf_cache
|
||||
TZ: Europe/Madrid
|
||||
volumes:
|
||||
- ./hf_cache:/app/hf_cache
|
||||
- ./models:/app/models
|
||||
networks:
|
||||
- backend
|
||||
profiles:
|
||||
- remote
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
memory: 6G
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: 1
|
||||
capabilities: [ gpu ]
|
||||
restart: unless-stopped
|
||||
|
||||
# ==================================================================================
|
||||
# TRANSLATION SCHEDULER - Creates translation jobs
|
||||
# ==================================================================================
|
||||
translation-scheduler:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile.scheduler
|
||||
image: rss2-scheduler:latest
|
||||
container_name: rss2_translation_scheduler
|
||||
environment:
|
||||
DB_HOST: db
|
||||
DB_PORT: 5432
|
||||
DB_NAME: ${DB_NAME:-rss}
|
||||
DB_USER: ${DB_USER:-rss}
|
||||
DB_PASS: ${DB_PASS}
|
||||
TARGET_LANGS: es
|
||||
SCHEDULER_BATCH: 1000
|
||||
SCHEDULER_SLEEP: 30
|
||||
TZ: Europe/Madrid
|
||||
volumes:
|
||||
- ./workers:/app/workers
|
||||
networks:
|
||||
- backend
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: '0.5'
|
||||
memory: 256M
|
||||
depends_on:
|
||||
db:
|
||||
condition: service_healthy
|
||||
restart: unless-stopped
|
||||
|
||||
embeddings:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile
|
||||
container_name: rss2_embeddings_py
|
||||
command: bash -lc "python -m workers.embeddings_worker"
|
||||
environment:
|
||||
DB_HOST: db
|
||||
DB_PORT: 5432
|
||||
DB_NAME: ${DB_NAME:-rss}
|
||||
DB_USER: ${DB_USER:-rss}
|
||||
DB_PASS: ${DB_PASS}
|
||||
EMB_MODEL: sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2
|
||||
EMB_BATCH: 64
|
||||
EMB_SLEEP_IDLE: 5
|
||||
EMB_LANGS: es
|
||||
EMB_LIMIT: 1000
|
||||
DEVICE: cuda
|
||||
HF_HOME: /app/hf_cache
|
||||
TZ: Europe/Madrid
|
||||
volumes:
|
||||
- ./workers:/app/workers
|
||||
- ./hf_cache:/app/hf_cache
|
||||
networks:
|
||||
- backend
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
memory: 6G
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: 1
|
||||
capabilities: [ gpu ]
|
||||
depends_on:
|
||||
db:
|
||||
condition: service_healthy
|
||||
restart: unless-stopped
|
||||
|
||||
# ==================================================================================
|
||||
# TOPICS WORKER (Go) - Matching temas y países
|
||||
# ==================================================================================
|
||||
topics:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile.topics
|
||||
container_name: rss2_topics
|
||||
environment:
|
||||
DB_HOST: db
|
||||
DB_PORT: 5432
|
||||
DB_NAME: ${DB_NAME:-rss}
|
||||
DB_USER: ${DB_USER:-rss}
|
||||
DB_PASS: ${DB_PASS}
|
||||
TOPICS_SLEEP: 10
|
||||
TOPICS_BATCH: 500
|
||||
TZ: Europe/Madrid
|
||||
networks:
|
||||
- backend
|
||||
depends_on:
|
||||
db:
|
||||
condition: service_healthy
|
||||
restart: unless-stopped
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: '1'
|
||||
memory: 512M
|
||||
|
||||
# ==================================================================================
|
||||
# RELATED WORKER (Go) - Noticias relacionadas
|
||||
# ==================================================================================
|
||||
related:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile.related
|
||||
container_name: rss2_related
|
||||
environment:
|
||||
DB_HOST: db
|
||||
DB_PORT: 5432
|
||||
DB_NAME: ${DB_NAME:-rss}
|
||||
DB_USER: ${DB_USER:-rss}
|
||||
DB_PASS: ${DB_PASS}
|
||||
RELATED_SLEEP: 10
|
||||
RELATED_BATCH: 200
|
||||
RELATED_TOPK: 10
|
||||
EMB_MODEL: mxbai-embed-large
|
||||
TZ: Europe/Madrid
|
||||
networks:
|
||||
- backend
|
||||
depends_on:
|
||||
db:
|
||||
condition: service_healthy
|
||||
restart: unless-stopped
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: '1'
|
||||
memory: 1G
|
||||
|
||||
qdrant:
|
||||
image: qdrant/qdrant:latest
|
||||
container_name: rss2_qdrant
|
||||
environment:
|
||||
TZ: Europe/Madrid
|
||||
QDRANT__SERVICE__GRPC_PORT: 6334
|
||||
volumes:
|
||||
- ./data/qdrant_storage:/qdrant/storage
|
||||
- /etc/timezone:/etc/timezone:ro
|
||||
- /etc/localtime:/etc/localtime:ro
|
||||
networks:
|
||||
- backend
|
||||
restart: unless-stopped
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: '4'
|
||||
memory: 4G
|
||||
reservations:
|
||||
memory: 2G
|
||||
|
||||
# ==================================================================================
|
||||
# QDRANT WORKER (Go) - Vectorización y búsqueda semántica
|
||||
# ==================================================================================
|
||||
qdrant-worker:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile.qdrant
|
||||
container_name: rss2_qdrant_worker
|
||||
environment:
|
||||
DB_HOST: db
|
||||
DB_PORT: 5432
|
||||
DB_NAME: ${DB_NAME:-rss}
|
||||
DB_USER: ${DB_USER:-rss}
|
||||
DB_PASS: ${DB_PASS}
|
||||
QDRANT_HOST: qdrant
|
||||
QDRANT_PORT: 6333
|
||||
QDRANT_COLLECTION: news_vectors
|
||||
OLLAMA_URL: http://ollama:11434
|
||||
QDRANT_SLEEP: 30
|
||||
QDRANT_BATCH: 100
|
||||
TZ: Europe/Madrid
|
||||
networks:
|
||||
- backend
|
||||
depends_on:
|
||||
db:
|
||||
condition: service_healthy
|
||||
qdrant:
|
||||
condition: service_started
|
||||
restart: unless-stopped
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: '1'
|
||||
memory: 1G
|
||||
|
||||
# ==================================================================================
|
||||
# NER WORKER (Python) - Extracción de entidades
|
||||
# ==================================================================================
|
||||
ner:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile
|
||||
container_name: rss2_ner
|
||||
command: bash -lc "python -m workers.ner_worker"
|
||||
environment:
|
||||
DB_HOST: db
|
||||
DB_PORT: 5432
|
||||
DB_NAME: ${DB_NAME:-rss}
|
||||
DB_USER: ${DB_USER:-rss}
|
||||
DB_PASS: ${DB_PASS}
|
||||
NER_LANG: es
|
||||
NER_BATCH: 64
|
||||
HF_HOME: /app/hf_cache
|
||||
TZ: Europe/Madrid
|
||||
volumes:
|
||||
- ./workers:/app/workers
|
||||
- ./hf_cache:/app/hf_cache
|
||||
networks:
|
||||
- backend
|
||||
depends_on:
|
||||
db:
|
||||
condition: service_healthy
|
||||
restart: unless-stopped
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: '2'
|
||||
memory: 2G
|
||||
|
||||
# ==================================================================================
|
||||
# CLUSTER WORKER (Python) - Agrupación de noticias
|
||||
# ==================================================================================
|
||||
cluster:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile
|
||||
container_name: rss2_cluster_py
|
||||
command: bash -lc "python -m workers.cluster_worker"
|
||||
environment:
|
||||
DB_HOST: db
|
||||
DB_PORT: 5432
|
||||
DB_NAME: ${DB_NAME:-rss}
|
||||
DB_USER: ${DB_USER:-rss}
|
||||
DB_PASS: ${DB_PASS}
|
||||
EVENT_DIST_THRESHOLD: 0.35
|
||||
EMB_MODEL: sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2
|
||||
TZ: Europe/Madrid
|
||||
volumes:
|
||||
- ./workers:/app/workers
|
||||
networks:
|
||||
- backend
|
||||
depends_on:
|
||||
db:
|
||||
condition: service_healthy
|
||||
restart: unless-stopped
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: '2'
|
||||
memory: 2G
|
||||
|
||||
# ==================================================================================
|
||||
# LLM CATEGORIZER (Python) - Categorización con Ollama
|
||||
# ==================================================================================
|
||||
llm-categorizer:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile
|
||||
container_name: rss2_llm_categorizer
|
||||
command: bash -lc "python -m workers.simple_categorizer_worker"
|
||||
environment:
|
||||
DB_HOST: db
|
||||
DB_PORT: 5432
|
||||
DB_NAME: ${DB_NAME:-rss}
|
||||
DB_USER: ${DB_USER:-rss}
|
||||
DB_PASS: ${DB_PASS}
|
||||
CATEGORIZER_BATCH_SIZE: 10
|
||||
CATEGORIZER_SLEEP_IDLE: 5
|
||||
TZ: Europe/Madrid
|
||||
volumes:
|
||||
- ./workers:/app/workers
|
||||
networks:
|
||||
- backend
|
||||
depends_on:
|
||||
db:
|
||||
condition: service_healthy
|
||||
restart: unless-stopped
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: '2'
|
||||
memory: 1G
|
||||
|
||||
# ==================================================================================
|
||||
# MONITORING STACK - SECURED
|
||||
# ==================================================================================
|
||||
|
||||
prometheus:
|
||||
image: prom/prometheus:latest
|
||||
container_name: rss2_prometheus
|
||||
volumes:
|
||||
- ./monitoring/prometheus.yml:/etc/prometheus/prometheus.yml:ro
|
||||
- prometheus_data:/prometheus
|
||||
command:
|
||||
- '--config.file=/etc/prometheus/prometheus.yml'
|
||||
- '--storage.tsdb.path=/prometheus'
|
||||
- '--web.console.libraries=/usr/share/prometheus/console_libraries'
|
||||
- '--web.console.templates=/usr/share/prometheus/consoles'
|
||||
# SEGURIDAD: Sin exposición de puertos - acceso solo vía Grafana o túnel SSH
|
||||
# ports:
|
||||
# - "9090:9090"
|
||||
networks:
|
||||
- monitoring
|
||||
restart: unless-stopped
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: '1'
|
||||
memory: 2G
|
||||
|
||||
grafana:
|
||||
image: grafana/grafana:latest
|
||||
container_name: rss2_grafana
|
||||
# SEGURIDAD: Acceso solo en localhost o vía túnel SSH
|
||||
# Para acceso remoto, usar túnel SSH: ssh -L 3001:localhost:3001 user@server
|
||||
ports:
|
||||
- "127.0.0.1:3001:3000"
|
||||
environment:
|
||||
# SEGURIDAD: Cambiar este password en producción
|
||||
- GF_SECURITY_ADMIN_PASSWORD=${GRAFANA_PASSWORD:-change_this_password}
|
||||
- GF_USERS_ALLOW_SIGN_UP=false
|
||||
- GF_SERVER_ROOT_URL=http://localhost:3001
|
||||
- GF_SECURITY_COOKIE_SECURE=false
|
||||
- GF_SECURITY_COOKIE_SAMESITE=lax
|
||||
volumes:
|
||||
- grafana_data:/var/lib/grafana
|
||||
networks:
|
||||
- monitoring
|
||||
depends_on:
|
||||
- prometheus
|
||||
restart: unless-stopped
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: '1'
|
||||
memory: 1G
|
||||
|
||||
cadvisor:
|
||||
image: gcr.io/cadvisor/cadvisor:latest
|
||||
container_name: rss2_cadvisor
|
||||
# SEGURIDAD: Sin exposición de puertos - solo acceso interno
|
||||
# ports:
|
||||
# - "8081:8080"
|
||||
volumes:
|
||||
- /:/rootfs:ro
|
||||
- /var/run:/var/run:ro
|
||||
- /sys:/sys:ro
|
||||
- /var/lib/docker/:/var/lib/docker:ro
|
||||
- /dev/disk/:/dev/disk:ro
|
||||
devices:
|
||||
- /dev/kmsg
|
||||
networks:
|
||||
- monitoring
|
||||
restart: unless-stopped
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: '0.5'
|
||||
memory: 512M
|
||||
|
||||
# ==================================================================================
|
||||
# REDES SEGMENTADAS
|
||||
# ==================================================================================
|
||||
networks:
|
||||
# Red frontal - Solo nginx y web app
|
||||
frontend:
|
||||
name: rss2_frontend
|
||||
driver: bridge
|
||||
internal: false
|
||||
|
||||
# Red backend - Base de datos, workers, redis, qdrant
|
||||
backend:
|
||||
name: rss2_backend
|
||||
driver: bridge
|
||||
internal: false # Acceso externo permitido (necesario para ingestor)
|
||||
|
||||
# Red de monitoreo - Prometheus, Grafana, cAdvisor
|
||||
monitoring:
|
||||
name: rss2_monitoring
|
||||
driver: bridge
|
||||
internal: true
|
||||
|
||||
volumes:
|
||||
prometheus_data:
|
||||
grafana_data:
|
||||
torch_extensions:
|
||||
|
|
@ -1,38 +0,0 @@
|
|||
#!/bin/bash
|
||||
set -e
|
||||
|
||||
# Directorio de datos de PostgreSQL
|
||||
PGDATA_DIR="/var/lib/postgresql/data/18/main"
|
||||
|
||||
echo "RSS2: Checking database presence..."
|
||||
|
||||
# REGLA: nunca se borra ni se modifica el directorio de datos automáticamente.
|
||||
# Solo se informa del estado y se deja que el entrypoint oficial de PostgreSQL
|
||||
# inicialice (si no existe) o arranque con los datos existentes.
|
||||
if [ -f "$PGDATA_DIR/PG_VERSION" ]; then
|
||||
echo "RSS2: Existing database found at $PGDATA_DIR - starting normally"
|
||||
else
|
||||
echo "RSS2: No database found at $PGDATA_DIR - docker-entrypoint will initialize it"
|
||||
fi
|
||||
|
||||
# Ejecutar el entrypoint original con los parámetros de PostgreSQL.
|
||||
# Si la base de datos estuviera corrupta, PostgreSQL fallará al arrancar y
|
||||
# registrará el error en los logs; NO se elimina ningún dato de forma automática.
|
||||
exec docker-entrypoint.sh \
|
||||
postgres \
|
||||
-c max_connections=200 \
|
||||
-c shared_buffers=4GB \
|
||||
-c effective_cache_size=12GB \
|
||||
-c work_mem=16MB \
|
||||
-c maintenance_work_mem=512MB \
|
||||
-c autovacuum_max_workers=3 \
|
||||
-c autovacuum_vacuum_scale_factor=0.02 \
|
||||
-c autovacuum_vacuum_cost_limit=1000 \
|
||||
-c max_worker_processes=8 \
|
||||
-c max_parallel_workers=6 \
|
||||
-c max_parallel_workers_per_gather=2 \
|
||||
-c wal_level=replica \
|
||||
-c max_wal_senders=5 \
|
||||
-c wal_keep_size=1GB \
|
||||
-c hot_standby=on \
|
||||
"$@"
|
||||
54
docs/DEPLOY.md
Normal file
54
docs/DEPLOY.md
Normal file
|
|
@ -0,0 +1,54 @@
|
|||
# Deployment Guide
|
||||
|
||||
This guide describes how to deploy the application to a new server.
|
||||
|
||||
## Prerequisites
|
||||
|
||||
* **Linux Server** (Ubuntu 22.04+ recommended)
|
||||
* **NVIDIA GPU**: Required for translation, embeddings, and NER services.
|
||||
* **NVIDIA Container Toolkit**: Must be installed to allow Docker to access the GPU.
|
||||
* **Docker** & **Docker Compose**: Latest versions.
|
||||
* **Git**: To clone the repository.
|
||||
* **External Service**: An instance of [AllTalk](https://github.com/erew123/alltalk_tts) running externally or on the host (port 7851 by default).
|
||||
|
||||
## Deployment Steps
|
||||
|
||||
1. **Clone the Repository**
|
||||
```bash
|
||||
git clone <your-repo-url>
|
||||
cd <your-repo-name>
|
||||
```
|
||||
|
||||
2. **Configure Environment Variables**
|
||||
Copy the example configuration file:
|
||||
```bash
|
||||
cp .env.example .env
|
||||
```
|
||||
Edit `.env` and set secure passwords and configuration:
|
||||
```bash
|
||||
nano .env
|
||||
```
|
||||
* Change `POSTGRES_PASSWORD` and `DB_PASS` to a strong unique password.
|
||||
* Change `SECRET_KEY` to a long random string.
|
||||
* Verify `ALLTALK_URL` points to your AllTalk instance (default assumes host machine access).
|
||||
|
||||
3. **Start the Services**
|
||||
Run the following command to build and start the application:
|
||||
```bash
|
||||
docker compose up -d --build
|
||||
```
|
||||
|
||||
4. **Database Initialization**
|
||||
The database will automatically initialize on the first run using the scripts in `init-db/`. This may take a few minutes. Check logs with:
|
||||
```bash
|
||||
docker compose logs -f db
|
||||
```
|
||||
|
||||
5. **Verify Deployment**
|
||||
Access the application at `http://<your-server-ip>:8001`.
|
||||
|
||||
## Important Notes
|
||||
|
||||
* **Models**: The application mounts `./models` and `./hf_cache` to persist AI models. On the first run, it will attempt to download necessary models (NLLB, BERT, etc.), which requires significant bandwidth and time.
|
||||
* **Data Persistence**: Database data is stored in `./pgdata` (mapped in docker-compose). Ensure this directory is backed up.
|
||||
* **Security**: Ensure port 5432 (Postgres) and 6379 (Redis) are firewall-protected and not exposed to the public internet unless intended (Docker maps them to the host network).
|
||||
287
docs/DEPLOY_DEBIAN.md
Normal file
287
docs/DEPLOY_DEBIAN.md
Normal file
|
|
@ -0,0 +1,287 @@
|
|||
# COCONEWS · Despliegue en Debian (sin Docker)
|
||||
|
||||
Guía completa para instalar y operar COCONEWS en un servidor Debian 12 (Bookworm) o Ubuntu 22.04+, sin Docker ni contenedores.
|
||||
|
||||
---
|
||||
|
||||
## Requisitos de hardware
|
||||
|
||||
| Modo | CPU | RAM | Disco |
|
||||
|------|-----|-----|-------|
|
||||
| **Mínimo (solo CPU)** | 4 cores | 8 GB | 40 GB |
|
||||
| **Recomendado** | 8 cores | 16 GB | 80 GB |
|
||||
| **Con GPU** | 8 cores + NVIDIA | 16 GB | 80 GB |
|
||||
|
||||
> Los modelos de IA (NLLB-200 + MiniLM + spaCy) ocupan ~5 GB en disco una vez descargados.
|
||||
|
||||
---
|
||||
|
||||
## Servicios que se instalan en el servidor
|
||||
|
||||
| Servicio | Tecnología | Gestionado por |
|
||||
|----------|-----------|----------------|
|
||||
| Base de datos | PostgreSQL 16 | apt + systemd |
|
||||
| Caché | Redis 7 | apt + systemd |
|
||||
| Búsqueda vectorial | Qdrant (binario) | systemd |
|
||||
| API REST | Go (backend) | systemd |
|
||||
| Ingestor RSS | Go | systemd |
|
||||
| Scraper / Discovery / Wiki / Topics / Related / Qdrant-worker | Go | systemd |
|
||||
| Traducción (NLLB-200) | Python + CTranslate2 | systemd |
|
||||
| Embeddings | Python + Sentence-Transformers | systemd |
|
||||
| NER | Python + spaCy | systemd |
|
||||
| Clustering / Categorización | Python | systemd |
|
||||
| Frontend | React (estático compilado) | nginx |
|
||||
| Proxy / Web | nginx | apt + systemd |
|
||||
|
||||
---
|
||||
|
||||
## Instalación paso a paso
|
||||
|
||||
### 1. Clonar el repositorio
|
||||
|
||||
```bash
|
||||
git clone https://gitea.laenre.net/pietre/rss2.git /opt/src/rss2
|
||||
cd /opt/src/rss2
|
||||
git checkout coconews
|
||||
```
|
||||
|
||||
### 2. Configurar variables de entorno
|
||||
|
||||
```bash
|
||||
cp deploy/debian/env.example /opt/rss2/.env
|
||||
nano /opt/rss2/.env
|
||||
```
|
||||
|
||||
Valores que **debes cambiar obligatoriamente**:
|
||||
|
||||
```env
|
||||
POSTGRES_PASSWORD=contraseña_segura_postgres
|
||||
DB_PASS=contraseña_segura_postgres
|
||||
REDIS_PASSWORD=contraseña_segura_redis
|
||||
SECRET_KEY=cadena_aleatoria_minimo_32_caracteres
|
||||
```
|
||||
|
||||
Genera claves seguras con:
|
||||
```bash
|
||||
openssl rand -hex 32
|
||||
```
|
||||
|
||||
### 3. Descargar y convertir el modelo de traducción (NLLB-200)
|
||||
|
||||
Este paso se hace **una sola vez** y puede tardar 10-30 minutos dependiendo de la conexión.
|
||||
|
||||
```bash
|
||||
# Instalar dependencias Python primero (si aun no se hizo)
|
||||
python3 -m venv /opt/rss2/venv
|
||||
/opt/rss2/venv/bin/pip install ctranslate2 transformers sentencepiece
|
||||
|
||||
# Convertir modelo NLLB-200 a formato CTranslate2 (tarda 10-30 min)
|
||||
mkdir -p /opt/rss2/models/nllb-ct2
|
||||
HF_HOME=/opt/rss2/hf_cache \
|
||||
/opt/rss2/venv/bin/ct2-transformers-converter \
|
||||
--model facebook/nllb-200-distilled-600M \
|
||||
--output_dir /opt/rss2/models/nllb-ct2 \
|
||||
--quantization int8 \
|
||||
--force
|
||||
|
||||
# Verificar que se generó correctamente
|
||||
ls /opt/rss2/models/nllb-ct2/model.bin && echo "Modelo OK"
|
||||
```
|
||||
|
||||
> El modelo ocupa ~600 MB convertido. Si la descarga de HuggingFace falla:
|
||||
> `export HF_ENDPOINT=https://huggingface.co` antes del comando de conversión.
|
||||
|
||||
> **Nota:** El worker convierte el modelo automáticamente si no lo encuentra,
|
||||
> pero hacerlo a mano evita que el primer arranque tarde 30 minutos.
|
||||
|
||||
### 4. Ejecutar el instalador
|
||||
|
||||
```bash
|
||||
sudo bash /opt/src/rss2/deploy/debian/install.sh
|
||||
```
|
||||
|
||||
El script hace automáticamente:
|
||||
- Instala PostgreSQL, Redis, nginx, Go, Node.js via apt
|
||||
- Descarga el binario de Qdrant
|
||||
- Crea el usuario `rss2` del sistema
|
||||
- Crea la base de datos y ejecuta las migraciones
|
||||
- Compila los 8 binarios Go
|
||||
- Compila el frontend React
|
||||
- Instala y habilita los 16 servicios systemd
|
||||
|
||||
---
|
||||
|
||||
## Verificar que funciona
|
||||
|
||||
```bash
|
||||
# Estado de todos los servicios
|
||||
systemctl status rss2-backend rss2-ingestor rss2-translator rss2-embeddings
|
||||
|
||||
# Ver logs en tiempo real
|
||||
journalctl -u rss2-backend -f
|
||||
journalctl -u rss2-translator -f
|
||||
|
||||
# Comprobar que el API responde
|
||||
curl http://localhost:8080/api/stats
|
||||
|
||||
# Acceder al frontend
|
||||
# http://IP_DEL_SERVIDOR:8001
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Gestión de servicios
|
||||
|
||||
### Iniciar / parar / reiniciar
|
||||
|
||||
```bash
|
||||
# Un servicio concreto
|
||||
systemctl start rss2-backend
|
||||
systemctl stop rss2-translator
|
||||
systemctl restart rss2-embeddings
|
||||
|
||||
# Todos los workers de una vez
|
||||
systemctl restart rss2-backend rss2-ingestor rss2-scraper rss2-discovery \
|
||||
rss2-wiki rss2-topics rss2-related rss2-qdrant-worker \
|
||||
rss2-langdetect rss2-translation-scheduler rss2-translator \
|
||||
rss2-embeddings rss2-ner rss2-cluster rss2-categorizer
|
||||
```
|
||||
|
||||
### Ver logs
|
||||
|
||||
```bash
|
||||
journalctl -u rss2-backend -f # API Go
|
||||
journalctl -u rss2-translator -f # Traductor
|
||||
journalctl -u rss2-embeddings -f # Embeddings
|
||||
journalctl -u rss2-ner -f # NER entidades
|
||||
journalctl -u rss2-ingestor -f # Ingestor RSS
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Actualizar el código
|
||||
|
||||
Cuando hay nuevos cambios en el repositorio:
|
||||
|
||||
```bash
|
||||
cd /opt/src/rss2
|
||||
git pull
|
||||
sudo bash deploy/debian/build.sh
|
||||
```
|
||||
|
||||
El script `build.sh` recompila los binarios Go, el frontend y sincroniza los workers Python, y reinicia los servicios automáticamente.
|
||||
|
||||
---
|
||||
|
||||
## Estructura de directorios en el servidor
|
||||
|
||||
```
|
||||
/opt/rss2/
|
||||
├── .env # Variables de entorno (permisos 600)
|
||||
├── bin/ # Binarios Go compilados
|
||||
│ ├── server # API REST
|
||||
│ ├── ingestor # Ingestor RSS
|
||||
│ ├── scraper
|
||||
│ ├── discovery
|
||||
│ ├── wiki_worker
|
||||
│ ├── topics
|
||||
│ ├── related
|
||||
│ └── qdrant_worker
|
||||
├── src/
|
||||
│ └── workers/ # Workers Python
|
||||
├── venv/ # Virtualenv Python (ML)
|
||||
├── models/
|
||||
│ └── nllb-ct2/ # Modelo traduccion CTranslate2
|
||||
├── hf_cache/ # Cache HuggingFace (embeddings, NER)
|
||||
├── frontend/
|
||||
│ └── dist/ # Frontend React compilado (servido por nginx)
|
||||
├── data/
|
||||
│ ├── wiki_images/ # Imagenes Wikipedia descargadas
|
||||
│ └── qdrant_storage/ # Datos vectoriales Qdrant
|
||||
└── qdrant/
|
||||
└── qdrant # Binario Qdrant
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Requisitos de red / firewall
|
||||
|
||||
Solo exponer al exterior el puerto **8001** (nginx). El resto deben ser internos:
|
||||
|
||||
```bash
|
||||
# Con ufw:
|
||||
ufw allow 8001/tcp # COCONEWS web
|
||||
ufw deny 5432/tcp # PostgreSQL - solo localhost
|
||||
ufw deny 6379/tcp # Redis - solo localhost
|
||||
ufw deny 6333/tcp # Qdrant - solo localhost
|
||||
ufw deny 8080/tcp # API Go - solo localhost (nginx hace proxy)
|
||||
ufw enable
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Backup de datos
|
||||
|
||||
```bash
|
||||
# PostgreSQL
|
||||
sudo -u postgres pg_dump rss > /opt/rss2/backups/rss_$(date +%Y%m%d).sql
|
||||
|
||||
# Datos Qdrant
|
||||
systemctl stop rss2-qdrant
|
||||
tar -czf /opt/rss2/backups/qdrant_$(date +%Y%m%d).tar.gz /opt/rss2/data/qdrant_storage
|
||||
systemctl start rss2-qdrant
|
||||
|
||||
# Imagenes Wikipedia (opcional, se pueden re-descargar)
|
||||
tar -czf /opt/rss2/backups/wiki_images_$(date +%Y%m%d).tar.gz /opt/rss2/data/wiki_images
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Solución de problemas frecuentes
|
||||
|
||||
### El traductor no arranca
|
||||
|
||||
```bash
|
||||
journalctl -u rss2-translator -n 50
|
||||
# Si dice "model not found": el modelo NLLB-200 no está convertido
|
||||
# Ejecutar el paso 3 de la instalación
|
||||
```
|
||||
|
||||
### PostgreSQL rechaza la conexión
|
||||
|
||||
```bash
|
||||
# Verificar que el .env tiene DB_HOST=127.0.0.1 (no "db")
|
||||
grep DB_HOST /opt/rss2/.env
|
||||
|
||||
# Verificar que el usuario existe
|
||||
sudo -u postgres psql -c "\du"
|
||||
```
|
||||
|
||||
### nginx devuelve 502 Bad Gateway
|
||||
|
||||
```bash
|
||||
# El backend Go no está corriendo
|
||||
systemctl status rss2-backend
|
||||
journalctl -u rss2-backend -n 30
|
||||
```
|
||||
|
||||
### Memoria insuficiente para los modelos Python
|
||||
|
||||
Con 8 GB RAM el translator + embeddings + NER pueden coincidir. Si el servidor tiene poca RAM, deshabilitar el translator-gpu y bajar el batch:
|
||||
|
||||
```bash
|
||||
# En /opt/rss2/.env
|
||||
TRANSLATOR_BATCH=8
|
||||
EMB_BATCH=32
|
||||
NER_BATCH=16
|
||||
systemctl restart rss2-translator rss2-embeddings rss2-ner
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Primer inicio de sesión
|
||||
|
||||
1. Abrir `http://IP:8001` en el navegador
|
||||
2. Al no haber usuarios, el sistema te redirige al registro
|
||||
3. El primer usuario registrado se convierte en **administrador**
|
||||
4. Ir a Configuración → Feeds → Importar el `feeds.csv` del repositorio para empezar con fuentes precargadas
|
||||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Add table
Add a link
Reference in a new issue