feat(deploy): despliegue nativo Debian sin Docker
- Elimina todos los Dockerfiles y docker-compose.yml
- Elimina scripts Docker (start_docker, reset_and_deploy, deploy-clean)
- Agrega deploy/debian/ con despliegue nativo via systemd:
- install.sh: instalacion completa en Debian (PostgreSQL, Redis,
Qdrant binario, Go, Python venv, nginx, frontend compilado)
- build.sh: recompila binarios Go y frontend sin reinstalar
- env.example: variables de entorno sin referencias Docker
- nginx.conf: sirve React estatico + proxy al API Go en localhost
- systemd/*.service: 16 servicios (8 Go + 7 Python + Qdrant)
Todos los hostnames Docker (db, redis, qdrant) reemplazados por 127.0.0.1
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
SITO
parent
ee90335b92
commit
10d51c3c52
39 changed files with 975 additions and 1319 deletions
69
deploy/debian/build.sh
Executable file
69
deploy/debian/build.sh
Executable file
|
|
@ -0,0 +1,69 @@
|
|||
#!/usr/bin/env bash
|
||||
# =============================================================================
|
||||
# RSS2 - Recompila binarios y frontend (sin reinstalar el sistema)
|
||||
# Usar despues de actualizar el codigo: bash build.sh
|
||||
# =============================================================================
|
||||
set -euo pipefail
|
||||
|
||||
RSS2_HOME="/opt/rss2"
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
|
||||
|
||||
export PATH=$PATH:/usr/local/go/bin
|
||||
|
||||
GREEN='\033[0;32m'; YELLOW='\033[1;33m'; NC='\033[0m'
|
||||
info() { echo -e "${GREEN}[BUILD]${NC} $*"; }
|
||||
warn() { echo -e "${YELLOW}[WARN]${NC} $*"; }
|
||||
|
||||
# --- Go Backend + Workers ---
|
||||
if [[ -d "$REPO_ROOT/backend" ]]; then
|
||||
info "Compilando backend Go..."
|
||||
(cd "$REPO_ROOT/backend" && \
|
||||
CGO_ENABLED=0 GOOS=linux go build -buildvcs=false -o "$RSS2_HOME/bin/server" ./cmd/server)
|
||||
info " [OK] server"
|
||||
|
||||
for cmd in scraper discovery wiki_worker topics related qdrant; do
|
||||
[[ -d "$REPO_ROOT/backend/cmd/$cmd" ]] || continue
|
||||
(cd "$REPO_ROOT/backend" && \
|
||||
CGO_ENABLED=0 GOOS=linux go build -buildvcs=false -o "$RSS2_HOME/bin/$cmd" "./cmd/$cmd")
|
||||
info " [OK] $cmd"
|
||||
done
|
||||
fi
|
||||
|
||||
# --- Ingestor Go ---
|
||||
if [[ -d "$REPO_ROOT/rss-ingestor-go" ]]; then
|
||||
info "Compilando ingestor Go..."
|
||||
(cd "$REPO_ROOT/rss-ingestor-go" && \
|
||||
CGO_ENABLED=0 GOOS=linux go build -buildvcs=false -o "$RSS2_HOME/bin/ingestor" .)
|
||||
info " [OK] ingestor"
|
||||
fi
|
||||
|
||||
# --- Frontend React ---
|
||||
if [[ -d "$REPO_ROOT/frontend" ]]; then
|
||||
info "Compilando frontend React..."
|
||||
(cd "$REPO_ROOT/frontend" && \
|
||||
npm install --silent && \
|
||||
VITE_API_URL=/api npm run build -- --outDir "$RSS2_HOME/frontend/dist")
|
||||
info " [OK] frontend"
|
||||
fi
|
||||
|
||||
# --- Workers Python ---
|
||||
info "Sincronizando workers Python..."
|
||||
rsync -a --delete "$REPO_ROOT/workers/" "$RSS2_HOME/src/workers/"
|
||||
cp "$REPO_ROOT/entity_config.json" "$RSS2_HOME/src/" 2>/dev/null || true
|
||||
info " [OK] workers Python"
|
||||
|
||||
chown -R rss2:rss2 "$RSS2_HOME/bin" "$RSS2_HOME/frontend/dist" "$RSS2_HOME/src"
|
||||
|
||||
# --- Restart servicios ---
|
||||
info "Reiniciando servicios..."
|
||||
GO_SERVICES=(rss2-backend rss2-ingestor rss2-scraper rss2-discovery rss2-wiki rss2-topics rss2-related rss2-qdrant-worker)
|
||||
PY_SERVICES=(rss2-langdetect rss2-translation-scheduler rss2-translator rss2-embeddings rss2-ner rss2-cluster rss2-categorizer)
|
||||
|
||||
for svc in "${GO_SERVICES[@]}" "${PY_SERVICES[@]}"; do
|
||||
systemctl is-active --quiet "$svc" && systemctl restart "$svc" && info " restarted $svc" || true
|
||||
done
|
||||
|
||||
systemctl reload nginx 2>/dev/null || true
|
||||
|
||||
info "Build completado."
|
||||
104
deploy/debian/env.example
Normal file
104
deploy/debian/env.example
Normal file
|
|
@ -0,0 +1,104 @@
|
|||
# =============================================================================
|
||||
# RSS2 - Variables de entorno para despliegue Debian nativo
|
||||
# Copiar a /opt/rss2/.env y editar valores antes de instalar
|
||||
# =============================================================================
|
||||
|
||||
# --- PostgreSQL ---
|
||||
POSTGRES_DB=rss
|
||||
POSTGRES_USER=rss
|
||||
POSTGRES_PASSWORD=CAMBIA_ESTO_postgres_password
|
||||
|
||||
# Usadas por workers Go (equivalente a DATABASE_URL)
|
||||
DB_HOST=127.0.0.1
|
||||
DB_PORT=5432
|
||||
DB_NAME=rss
|
||||
DB_USER=rss
|
||||
DB_PASS=CAMBIA_ESTO_postgres_password
|
||||
|
||||
# URL completa para backend API Go
|
||||
DATABASE_URL=postgres://rss:CAMBIA_ESTO_postgres_password@127.0.0.1:5432/rss?sslmode=disable
|
||||
|
||||
# --- Redis ---
|
||||
REDIS_PASSWORD=CAMBIA_ESTO_redis_password
|
||||
REDIS_URL=redis://:CAMBIA_ESTO_redis_password@127.0.0.1:6379
|
||||
|
||||
# --- JWT Secret (minimo 32 caracteres, aleatorio) ---
|
||||
SECRET_KEY=CAMBIA_ESTO_jwt_secret_muy_largo_y_aleatorio
|
||||
|
||||
# --- Backend API ---
|
||||
SERVER_PORT=8080
|
||||
|
||||
# --- Zona horaria ---
|
||||
TZ=Europe/Madrid
|
||||
|
||||
# --- HuggingFace cache (modelos ML) ---
|
||||
HF_HOME=/opt/rss2/hf_cache
|
||||
|
||||
# --- Qdrant (local, sin Docker) ---
|
||||
QDRANT_HOST=127.0.0.1
|
||||
QDRANT_PORT=6333
|
||||
QDRANT_COLLECTION=news_vectors
|
||||
|
||||
# --- Translator (NLLB-200 via CTranslate2) ---
|
||||
TARGET_LANGS=es
|
||||
TRANSLATOR_BATCH=32
|
||||
CT2_MODEL_PATH=/opt/rss2/models/nllb-ct2
|
||||
CT2_DEVICE=cpu
|
||||
CT2_COMPUTE_TYPE=int8
|
||||
UNIVERSAL_MODEL=facebook/nllb-200-distilled-600M
|
||||
|
||||
# --- Embeddings ---
|
||||
EMB_MODEL=sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2
|
||||
EMB_BATCH=64
|
||||
EMB_SLEEP_IDLE=5
|
||||
EMB_LANGS=es
|
||||
EMB_LIMIT=1000
|
||||
DEVICE=cpu
|
||||
|
||||
# --- NER ---
|
||||
NER_LANG=es
|
||||
NER_BATCH=64
|
||||
|
||||
# --- Ingestor RSS ---
|
||||
RSS_MAX_WORKERS=100
|
||||
RSS_POKE_INTERVAL_MIN=15
|
||||
|
||||
# --- Scraper ---
|
||||
SCRAPER_SLEEP=60
|
||||
SCRAPER_BATCH=10
|
||||
|
||||
# --- Discovery ---
|
||||
DISCOVERY_INTERVAL=900
|
||||
DISCOVERY_BATCH=10
|
||||
MAX_FEEDS_PER_URL=5
|
||||
|
||||
# --- Wiki Worker ---
|
||||
WIKI_SLEEP=10
|
||||
|
||||
# --- Topics ---
|
||||
TOPICS_SLEEP=10
|
||||
TOPICS_BATCH=500
|
||||
|
||||
# --- Related ---
|
||||
RELATED_SLEEP=10
|
||||
RELATED_BATCH=200
|
||||
RELATED_TOPK=10
|
||||
|
||||
# --- Cluster ---
|
||||
EVENT_DIST_THRESHOLD=0.35
|
||||
|
||||
# --- Categorizer ---
|
||||
CATEGORIZER_BATCH_SIZE=10
|
||||
CATEGORIZER_SLEEP_IDLE=5
|
||||
|
||||
# --- Scheduler traduccion ---
|
||||
SCHEDULER_BATCH=1000
|
||||
SCHEDULER_SLEEP=30
|
||||
|
||||
# --- Lang Detect ---
|
||||
LANG_DETECT_SLEEP=60
|
||||
LANG_DETECT_BATCH=1000
|
||||
|
||||
# --- Qdrant Worker ---
|
||||
QDRANT_SLEEP=30
|
||||
QDRANT_BATCH=100
|
||||
294
deploy/debian/install.sh
Executable file
294
deploy/debian/install.sh
Executable file
|
|
@ -0,0 +1,294 @@
|
|||
#!/usr/bin/env bash
|
||||
# =============================================================================
|
||||
# RSS2 - Instalacion en Debian (sin Docker)
|
||||
# Ejecutar como root: bash install.sh
|
||||
# =============================================================================
|
||||
set -euo pipefail
|
||||
|
||||
RSS2_USER="rss2"
|
||||
RSS2_HOME="/opt/rss2"
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
|
||||
|
||||
RED='\033[0;31m'; GREEN='\033[0;32m'; YELLOW='\033[1;33m'; NC='\033[0m'
|
||||
info() { echo -e "${GREEN}[INFO]${NC} $*"; }
|
||||
warn() { echo -e "${YELLOW}[WARN]${NC} $*"; }
|
||||
error() { echo -e "${RED}[ERROR]${NC} $*"; exit 1; }
|
||||
|
||||
[[ "$EUID" -ne 0 ]] && error "Ejecutar como root: sudo bash install.sh"
|
||||
|
||||
# =============================================================================
|
||||
# 1. DEPENDENCIAS DEL SISTEMA
|
||||
# =============================================================================
|
||||
info "Instalando dependencias del sistema..."
|
||||
apt-get update -qq
|
||||
apt-get install -y --no-install-recommends \
|
||||
curl wget git build-essential \
|
||||
postgresql postgresql-client \
|
||||
redis-server \
|
||||
nginx \
|
||||
python3 python3-pip python3-venv python3-dev \
|
||||
nodejs npm \
|
||||
ca-certificates tzdata \
|
||||
libpq-dev
|
||||
|
||||
# Go (si no esta instalado o version < 1.22)
|
||||
if ! command -v go &>/dev/null || [[ "$(go version | awk '{print $3}' | tr -d 'go')" < "1.22" ]]; then
|
||||
info "Instalando Go 1.23..."
|
||||
GO_VERSION="1.23.4"
|
||||
ARCH=$(dpkg --print-architecture)
|
||||
case "$ARCH" in
|
||||
amd64) GO_ARCH="amd64" ;;
|
||||
arm64) GO_ARCH="arm64" ;;
|
||||
*) error "Arquitectura no soportada: $ARCH" ;;
|
||||
esac
|
||||
curl -fsSL "https://go.dev/dl/go${GO_VERSION}.linux-${GO_ARCH}.tar.gz" -o /tmp/go.tar.gz
|
||||
rm -rf /usr/local/go
|
||||
tar -C /usr/local -xzf /tmp/go.tar.gz
|
||||
echo 'export PATH=$PATH:/usr/local/go/bin' > /etc/profile.d/go.sh
|
||||
export PATH=$PATH:/usr/local/go/bin
|
||||
rm /tmp/go.tar.gz
|
||||
fi
|
||||
info "Go: $(go version)"
|
||||
|
||||
# Qdrant (binario oficial)
|
||||
if [[ ! -f "$RSS2_HOME/qdrant/qdrant" ]]; then
|
||||
info "Descargando Qdrant..."
|
||||
QDRANT_VERSION="v1.12.1"
|
||||
ARCH=$(dpkg --print-architecture)
|
||||
case "$ARCH" in
|
||||
amd64) QDRANT_ARCH="x86_64-unknown-linux-musl" ;;
|
||||
arm64) QDRANT_ARCH="aarch64-unknown-linux-musl" ;;
|
||||
*) error "Arquitectura no soportada para Qdrant: $ARCH" ;;
|
||||
esac
|
||||
mkdir -p "$RSS2_HOME/qdrant"
|
||||
curl -fsSL "https://github.com/qdrant/qdrant/releases/download/${QDRANT_VERSION}/qdrant-${QDRANT_ARCH}.tar.gz" \
|
||||
-o /tmp/qdrant.tar.gz
|
||||
tar -C "$RSS2_HOME/qdrant" -xzf /tmp/qdrant.tar.gz
|
||||
chmod +x "$RSS2_HOME/qdrant/qdrant"
|
||||
rm /tmp/qdrant.tar.gz
|
||||
fi
|
||||
|
||||
# =============================================================================
|
||||
# 2. USUARIO Y DIRECTORIOS
|
||||
# =============================================================================
|
||||
info "Creando usuario $RSS2_USER y directorios..."
|
||||
id "$RSS2_USER" &>/dev/null || useradd -r -m -d "$RSS2_HOME" -s /bin/bash "$RSS2_USER"
|
||||
|
||||
mkdir -p \
|
||||
"$RSS2_HOME/bin" \
|
||||
"$RSS2_HOME/src" \
|
||||
"$RSS2_HOME/data/wiki_images" \
|
||||
"$RSS2_HOME/data/qdrant_storage" \
|
||||
"$RSS2_HOME/hf_cache" \
|
||||
"$RSS2_HOME/models" \
|
||||
"$RSS2_HOME/frontend/dist" \
|
||||
"$RSS2_HOME/logs"
|
||||
|
||||
# =============================================================================
|
||||
# 3. CONFIGURACION ENTORNO
|
||||
# =============================================================================
|
||||
if [[ ! -f "$RSS2_HOME/.env" ]]; then
|
||||
if [[ -f "$SCRIPT_DIR/env.example" ]]; then
|
||||
cp "$SCRIPT_DIR/env.example" "$RSS2_HOME/.env"
|
||||
warn "Copia env.example en $RSS2_HOME/.env - EDITA LAS CONTRASENAS antes de continuar"
|
||||
warn "Presiona Enter cuando hayas editado el .env, o Ctrl+C para salir"
|
||||
read -r
|
||||
else
|
||||
error "No se encontro env.example en $SCRIPT_DIR"
|
||||
fi
|
||||
fi
|
||||
|
||||
# =============================================================================
|
||||
# 4. POSTGRESQL
|
||||
# =============================================================================
|
||||
info "Configurando PostgreSQL..."
|
||||
source "$RSS2_HOME/.env" 2>/dev/null || true
|
||||
|
||||
DB_NAME="${POSTGRES_DB:-rss}"
|
||||
DB_USER="${POSTGRES_USER:-rss}"
|
||||
DB_PASS="${POSTGRES_PASSWORD:-changeme}"
|
||||
|
||||
systemctl enable --now postgresql
|
||||
|
||||
# Crear usuario y base de datos si no existen
|
||||
sudo -u postgres psql -tc "SELECT 1 FROM pg_roles WHERE rolname='$DB_USER'" | grep -q 1 || \
|
||||
sudo -u postgres psql -c "CREATE USER $DB_USER WITH PASSWORD '$DB_PASS';"
|
||||
|
||||
sudo -u postgres psql -tc "SELECT 1 FROM pg_database WHERE datname='$DB_NAME'" | grep -q 1 || \
|
||||
sudo -u postgres createdb -O "$DB_USER" "$DB_NAME"
|
||||
|
||||
# Ejecutar migraciones SQL
|
||||
if [[ -d "$REPO_ROOT/migrations" ]]; then
|
||||
info "Ejecutando migraciones..."
|
||||
for sql_file in "$REPO_ROOT/migrations"/*.sql; do
|
||||
[[ -f "$sql_file" ]] || continue
|
||||
info " Aplicando $(basename "$sql_file")..."
|
||||
sudo -u postgres psql -d "$DB_NAME" -f "$sql_file" 2>/dev/null || warn " (ya aplicada o error ignorado)"
|
||||
done
|
||||
fi
|
||||
|
||||
# Ejecutar init-db scripts (schema inicial)
|
||||
if [[ -d "$REPO_ROOT/init-db" ]]; then
|
||||
info "Ejecutando scripts de init-db..."
|
||||
for sql_file in "$REPO_ROOT/init-db"/*.sql; do
|
||||
[[ -f "$sql_file" ]] || continue
|
||||
info " $(basename "$sql_file")..."
|
||||
sudo -u postgres psql -d "$DB_NAME" -f "$sql_file" 2>/dev/null || warn " (ya aplicada o error ignorado)"
|
||||
done
|
||||
fi
|
||||
|
||||
# =============================================================================
|
||||
# 5. REDIS
|
||||
# =============================================================================
|
||||
info "Configurando Redis..."
|
||||
REDIS_PASS="${REDIS_PASSWORD:-changeme_redis}"
|
||||
|
||||
# Agregar autenticacion y limites de memoria a redis.conf
|
||||
REDIS_CONF="/etc/redis/redis.conf"
|
||||
grep -q "requirepass $REDIS_PASS" "$REDIS_CONF" 2>/dev/null || {
|
||||
echo "requirepass $REDIS_PASS" >> "$REDIS_CONF"
|
||||
echo "maxmemory 512mb" >> "$REDIS_CONF"
|
||||
echo "maxmemory-policy allkeys-lru" >> "$REDIS_CONF"
|
||||
echo "appendonly yes" >> "$REDIS_CONF"
|
||||
}
|
||||
systemctl enable --now redis-server
|
||||
|
||||
# =============================================================================
|
||||
# 6. PYTHON VIRTUALENV + DEPENDENCIAS ML
|
||||
# =============================================================================
|
||||
info "Creando virtualenv Python y instalando dependencias..."
|
||||
python3 -m venv "$RSS2_HOME/venv"
|
||||
"$RSS2_HOME/venv/bin/pip" install --upgrade pip -q
|
||||
|
||||
if [[ -f "$REPO_ROOT/requirements.txt" ]]; then
|
||||
"$RSS2_HOME/venv/bin/pip" install -r "$REPO_ROOT/requirements.txt" -q
|
||||
fi
|
||||
|
||||
# spaCy modelo en español
|
||||
"$RSS2_HOME/venv/bin/python" -m spacy download es_core_news_lg 2>/dev/null || \
|
||||
warn "spaCy model es_core_news_lg no se pudo descargar, hazlo manualmente"
|
||||
|
||||
# Copiar workers Python al directorio de trabajo
|
||||
info "Copiando workers Python..."
|
||||
rsync -a --delete "$REPO_ROOT/workers/" "$RSS2_HOME/src/workers/"
|
||||
cp "$REPO_ROOT/entity_config.json" "$RSS2_HOME/src/" 2>/dev/null || true
|
||||
|
||||
# =============================================================================
|
||||
# 7. COMPILAR GO (backend + workers)
|
||||
# =============================================================================
|
||||
info "Compilando binarios Go..."
|
||||
export PATH=$PATH:/usr/local/go/bin
|
||||
export GOPATH=/tmp/go-build-rss2
|
||||
|
||||
# Backend API
|
||||
if [[ -d "$REPO_ROOT/backend" ]]; then
|
||||
(cd "$REPO_ROOT/backend" && \
|
||||
CGO_ENABLED=0 GOOS=linux go build -buildvcs=false -o "$RSS2_HOME/bin/server" ./cmd/server && \
|
||||
info " [OK] server") || warn " [FAIL] server"
|
||||
for cmd in scraper discovery wiki_worker topics related qdrant; do
|
||||
[[ -d "$REPO_ROOT/backend/cmd/$cmd" ]] || continue
|
||||
(cd "$REPO_ROOT/backend" && \
|
||||
CGO_ENABLED=0 GOOS=linux go build -buildvcs=false -o "$RSS2_HOME/bin/$cmd" "./cmd/$cmd" && \
|
||||
info " [OK] $cmd") || warn " [FAIL] $cmd"
|
||||
done
|
||||
fi
|
||||
|
||||
# RSS Ingestor Go (repo separado)
|
||||
if [[ -d "$REPO_ROOT/rss-ingestor-go" ]]; then
|
||||
(cd "$REPO_ROOT/rss-ingestor-go" && \
|
||||
CGO_ENABLED=0 GOOS=linux go build -buildvcs=false -o "$RSS2_HOME/bin/ingestor" . && \
|
||||
info " [OK] ingestor") || warn " [FAIL] ingestor"
|
||||
fi
|
||||
|
||||
# =============================================================================
|
||||
# 8. FRONTEND REACT
|
||||
# =============================================================================
|
||||
info "Compilando frontend React..."
|
||||
if [[ -d "$REPO_ROOT/frontend" ]]; then
|
||||
(cd "$REPO_ROOT/frontend" && \
|
||||
npm install --silent && \
|
||||
VITE_API_URL=/api npm run build -- --outDir "$RSS2_HOME/frontend/dist" && \
|
||||
info " [OK] frontend compilado") || warn " [FAIL] frontend"
|
||||
fi
|
||||
|
||||
# =============================================================================
|
||||
# 9. NGINX
|
||||
# =============================================================================
|
||||
info "Configurando Nginx..."
|
||||
cp "$SCRIPT_DIR/nginx.conf" /etc/nginx/nginx.conf
|
||||
nginx -t && systemctl enable --now nginx && systemctl reload nginx
|
||||
|
||||
# =============================================================================
|
||||
# 10. SYSTEMD SERVICES
|
||||
# =============================================================================
|
||||
info "Instalando servicios systemd..."
|
||||
SERVICES=(
|
||||
rss2-qdrant
|
||||
rss2-backend
|
||||
rss2-ingestor
|
||||
rss2-scraper
|
||||
rss2-discovery
|
||||
rss2-wiki
|
||||
rss2-topics
|
||||
rss2-related
|
||||
rss2-qdrant-worker
|
||||
rss2-langdetect
|
||||
rss2-translation-scheduler
|
||||
rss2-translator
|
||||
rss2-embeddings
|
||||
rss2-ner
|
||||
rss2-cluster
|
||||
rss2-categorizer
|
||||
)
|
||||
|
||||
for svc in "${SERVICES[@]}"; do
|
||||
svc_file="$SCRIPT_DIR/systemd/${svc}.service"
|
||||
if [[ -f "$svc_file" ]]; then
|
||||
cp "$svc_file" "/etc/systemd/system/${svc}.service"
|
||||
else
|
||||
warn "No se encontro $svc_file"
|
||||
fi
|
||||
done
|
||||
|
||||
systemctl daemon-reload
|
||||
|
||||
for svc in "${SERVICES[@]}"; do
|
||||
systemctl enable "$svc" 2>/dev/null || true
|
||||
done
|
||||
|
||||
# =============================================================================
|
||||
# 11. PERMISOS FINALES
|
||||
# =============================================================================
|
||||
info "Ajustando permisos..."
|
||||
chown -R "$RSS2_USER:$RSS2_USER" "$RSS2_HOME"
|
||||
chmod 600 "$RSS2_HOME/.env"
|
||||
|
||||
# =============================================================================
|
||||
# 12. ARRANCAR SERVICIOS
|
||||
# =============================================================================
|
||||
info "Arrancando servicios..."
|
||||
# Infraestructura primero
|
||||
systemctl start rss2-qdrant
|
||||
sleep 3
|
||||
|
||||
# API y workers Go
|
||||
for svc in rss2-backend rss2-ingestor rss2-scraper rss2-discovery rss2-wiki rss2-topics rss2-related rss2-qdrant-worker; do
|
||||
systemctl start "$svc" || warn "No se pudo arrancar $svc"
|
||||
done
|
||||
|
||||
# Workers Python (modelos pesados, arrancan despues)
|
||||
for svc in rss2-langdetect rss2-translation-scheduler rss2-translator rss2-embeddings rss2-ner rss2-cluster rss2-categorizer; do
|
||||
systemctl start "$svc" || warn "No se pudo arrancar $svc"
|
||||
done
|
||||
|
||||
# =============================================================================
|
||||
echo ""
|
||||
info "============================================="
|
||||
info " RSS2 instalado en $RSS2_HOME"
|
||||
info " Acceder en: http://$(hostname -I | awk '{print $1}'):8001"
|
||||
info ""
|
||||
info " Ver logs: journalctl -u rss2-backend -f"
|
||||
info " Ver estado: systemctl status rss2-backend"
|
||||
info " Editar env: nano $RSS2_HOME/.env"
|
||||
info "============================================="
|
||||
91
deploy/debian/nginx.conf
Normal file
91
deploy/debian/nginx.conf
Normal file
|
|
@ -0,0 +1,91 @@
|
|||
user www-data;
|
||||
worker_processes auto;
|
||||
error_log /var/log/nginx/error.log warn;
|
||||
pid /run/nginx.pid;
|
||||
|
||||
events {
|
||||
worker_connections 2048;
|
||||
use epoll;
|
||||
}
|
||||
|
||||
http {
|
||||
include /etc/nginx/mime.types;
|
||||
default_type application/octet-stream;
|
||||
|
||||
log_format main '$remote_addr - $remote_user [$time_local] "$request" '
|
||||
'$status $body_bytes_sent "$http_referer" '
|
||||
'"$http_user_agent" "$http_x_forwarded_for"';
|
||||
|
||||
access_log /var/log/nginx/access.log main;
|
||||
|
||||
sendfile on;
|
||||
tcp_nopush on;
|
||||
tcp_nodelay on;
|
||||
keepalive_timeout 65;
|
||||
types_hash_max_size 2048;
|
||||
client_max_body_size 100M;
|
||||
|
||||
gzip on;
|
||||
gzip_vary on;
|
||||
gzip_proxied any;
|
||||
gzip_comp_level 6;
|
||||
gzip_types text/plain text/css text/javascript
|
||||
application/json application/javascript
|
||||
application/xml text/xml;
|
||||
|
||||
# Go API backend (proceso nativo en localhost)
|
||||
upstream api_backend {
|
||||
server 127.0.0.1:8080;
|
||||
keepalive 32;
|
||||
}
|
||||
|
||||
server {
|
||||
listen 8001;
|
||||
server_name _;
|
||||
|
||||
client_body_timeout 60s;
|
||||
client_header_timeout 60s;
|
||||
send_timeout 300s;
|
||||
|
||||
# Frontend React (archivos estaticos compilados)
|
||||
root /opt/rss2/frontend/dist;
|
||||
index index.html;
|
||||
|
||||
location / {
|
||||
try_files $uri $uri/ /index.html;
|
||||
}
|
||||
|
||||
# Imagenes Wikipedia servidas directamente
|
||||
location /wiki-images/ {
|
||||
alias /opt/rss2/data/wiki_images/;
|
||||
expires 7d;
|
||||
add_header Cache-Control "public, immutable";
|
||||
}
|
||||
|
||||
# Proxy al API Go
|
||||
location /api/ {
|
||||
proxy_pass http://api_backend/api/;
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
proxy_set_header Connection "";
|
||||
|
||||
proxy_connect_timeout 60s;
|
||||
proxy_send_timeout 300s;
|
||||
proxy_read_timeout 300s;
|
||||
}
|
||||
|
||||
location /health {
|
||||
access_log off;
|
||||
return 200 "ok";
|
||||
}
|
||||
|
||||
location ~ /\. {
|
||||
deny all;
|
||||
access_log off;
|
||||
log_not_found off;
|
||||
}
|
||||
}
|
||||
}
|
||||
24
deploy/debian/systemd/rss2-backend.service
Normal file
24
deploy/debian/systemd/rss2-backend.service
Normal file
|
|
@ -0,0 +1,24 @@
|
|||
[Unit]
|
||||
Description=RSS2 Backend API (Go)
|
||||
After=network.target postgresql.service redis.service
|
||||
Requires=postgresql.service redis.service
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=rss2
|
||||
Group=rss2
|
||||
WorkingDirectory=/opt/rss2
|
||||
EnvironmentFile=/opt/rss2/.env
|
||||
ExecStart=/opt/rss2/bin/server
|
||||
Restart=always
|
||||
RestartSec=5
|
||||
StandardOutput=journal
|
||||
StandardError=journal
|
||||
SyslogIdentifier=rss2-backend
|
||||
|
||||
# Limites de recursos
|
||||
LimitNOFILE=65536
|
||||
MemoryMax=1G
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
25
deploy/debian/systemd/rss2-categorizer.service
Normal file
25
deploy/debian/systemd/rss2-categorizer.service
Normal file
|
|
@ -0,0 +1,25 @@
|
|||
[Unit]
|
||||
Description=RSS2 Categorizer Worker (Python)
|
||||
After=network.target postgresql.service
|
||||
Requires=postgresql.service
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=rss2
|
||||
Group=rss2
|
||||
WorkingDirectory=/opt/rss2/src
|
||||
EnvironmentFile=/opt/rss2/.env
|
||||
Environment=CATEGORIZER_BATCH_SIZE=10
|
||||
Environment=CATEGORIZER_SLEEP_IDLE=5
|
||||
ExecStart=/opt/rss2/venv/bin/python -m workers.simple_categorizer_worker
|
||||
Restart=always
|
||||
RestartSec=10
|
||||
StandardOutput=journal
|
||||
StandardError=journal
|
||||
SyslogIdentifier=rss2-categorizer
|
||||
|
||||
MemoryMax=1G
|
||||
CPUQuota=200%
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
25
deploy/debian/systemd/rss2-cluster.service
Normal file
25
deploy/debian/systemd/rss2-cluster.service
Normal file
|
|
@ -0,0 +1,25 @@
|
|||
[Unit]
|
||||
Description=RSS2 Cluster Worker - Agrupacion de noticias (Python)
|
||||
After=network.target postgresql.service
|
||||
Requires=postgresql.service
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=rss2
|
||||
Group=rss2
|
||||
WorkingDirectory=/opt/rss2/src
|
||||
EnvironmentFile=/opt/rss2/.env
|
||||
Environment=EVENT_DIST_THRESHOLD=0.35
|
||||
Environment=EMB_MODEL=sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2
|
||||
ExecStart=/opt/rss2/venv/bin/python -m workers.cluster_worker
|
||||
Restart=always
|
||||
RestartSec=10
|
||||
StandardOutput=journal
|
||||
StandardError=journal
|
||||
SyslogIdentifier=rss2-cluster
|
||||
|
||||
MemoryMax=2G
|
||||
CPUQuota=200%
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
26
deploy/debian/systemd/rss2-discovery.service
Normal file
26
deploy/debian/systemd/rss2-discovery.service
Normal file
|
|
@ -0,0 +1,26 @@
|
|||
[Unit]
|
||||
Description=RSS2 Discovery de Feeds (Go)
|
||||
After=network.target postgresql.service
|
||||
Requires=postgresql.service
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=rss2
|
||||
Group=rss2
|
||||
WorkingDirectory=/opt/rss2
|
||||
EnvironmentFile=/opt/rss2/.env
|
||||
Environment=DISCOVERY_INTERVAL=900
|
||||
Environment=DISCOVERY_BATCH=10
|
||||
Environment=MAX_FEEDS_PER_URL=5
|
||||
ExecStart=/opt/rss2/bin/discovery
|
||||
Restart=always
|
||||
RestartSec=10
|
||||
StandardOutput=journal
|
||||
StandardError=journal
|
||||
SyslogIdentifier=rss2-discovery
|
||||
|
||||
MemoryMax=512M
|
||||
CPUQuota=100%
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
30
deploy/debian/systemd/rss2-embeddings.service
Normal file
30
deploy/debian/systemd/rss2-embeddings.service
Normal file
|
|
@ -0,0 +1,30 @@
|
|||
[Unit]
|
||||
Description=RSS2 Embeddings Worker (Python)
|
||||
After=network.target postgresql.service
|
||||
Requires=postgresql.service
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=rss2
|
||||
Group=rss2
|
||||
WorkingDirectory=/opt/rss2/src
|
||||
EnvironmentFile=/opt/rss2/.env
|
||||
Environment=EMB_MODEL=sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2
|
||||
Environment=EMB_BATCH=64
|
||||
Environment=EMB_SLEEP_IDLE=5
|
||||
Environment=EMB_LANGS=es
|
||||
Environment=EMB_LIMIT=1000
|
||||
Environment=DEVICE=cpu
|
||||
Environment=HF_HOME=/opt/rss2/hf_cache
|
||||
ExecStart=/opt/rss2/venv/bin/python -m workers.embeddings_worker
|
||||
Restart=always
|
||||
RestartSec=10
|
||||
StandardOutput=journal
|
||||
StandardError=journal
|
||||
SyslogIdentifier=rss2-embeddings
|
||||
|
||||
MemoryMax=3G
|
||||
CPUQuota=200%
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
26
deploy/debian/systemd/rss2-ingestor.service
Normal file
26
deploy/debian/systemd/rss2-ingestor.service
Normal file
|
|
@ -0,0 +1,26 @@
|
|||
[Unit]
|
||||
Description=RSS2 Ingestor RSS (Go)
|
||||
After=network.target postgresql.service
|
||||
Requires=postgresql.service
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=rss2
|
||||
Group=rss2
|
||||
WorkingDirectory=/opt/rss2
|
||||
EnvironmentFile=/opt/rss2/.env
|
||||
Environment=RSS_MAX_WORKERS=100
|
||||
Environment=RSS_POKE_INTERVAL_MIN=15
|
||||
ExecStart=/opt/rss2/bin/ingestor
|
||||
Restart=always
|
||||
RestartSec=10
|
||||
StandardOutput=journal
|
||||
StandardError=journal
|
||||
SyslogIdentifier=rss2-ingestor
|
||||
|
||||
LimitNOFILE=65536
|
||||
MemoryMax=2G
|
||||
CPUQuota=200%
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
25
deploy/debian/systemd/rss2-langdetect.service
Normal file
25
deploy/debian/systemd/rss2-langdetect.service
Normal file
|
|
@ -0,0 +1,25 @@
|
|||
[Unit]
|
||||
Description=RSS2 Language Detection Worker (Python)
|
||||
After=network.target postgresql.service
|
||||
Requires=postgresql.service
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=rss2
|
||||
Group=rss2
|
||||
WorkingDirectory=/opt/rss2/src
|
||||
EnvironmentFile=/opt/rss2/.env
|
||||
Environment=LANG_DETECT_SLEEP=60
|
||||
Environment=LANG_DETECT_BATCH=1000
|
||||
ExecStart=/opt/rss2/venv/bin/python -m workers.langdetect_worker
|
||||
Restart=always
|
||||
RestartSec=10
|
||||
StandardOutput=journal
|
||||
StandardError=journal
|
||||
SyslogIdentifier=rss2-langdetect
|
||||
|
||||
MemoryMax=512M
|
||||
CPUQuota=50%
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
26
deploy/debian/systemd/rss2-ner.service
Normal file
26
deploy/debian/systemd/rss2-ner.service
Normal file
|
|
@ -0,0 +1,26 @@
|
|||
[Unit]
|
||||
Description=RSS2 NER Worker - Extraccion de Entidades (Python)
|
||||
After=network.target postgresql.service
|
||||
Requires=postgresql.service
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=rss2
|
||||
Group=rss2
|
||||
WorkingDirectory=/opt/rss2/src
|
||||
EnvironmentFile=/opt/rss2/.env
|
||||
Environment=NER_LANG=es
|
||||
Environment=NER_BATCH=64
|
||||
Environment=HF_HOME=/opt/rss2/hf_cache
|
||||
ExecStart=/opt/rss2/venv/bin/python -m workers.ner_worker
|
||||
Restart=always
|
||||
RestartSec=10
|
||||
StandardOutput=journal
|
||||
StandardError=journal
|
||||
SyslogIdentifier=rss2-ner
|
||||
|
||||
MemoryMax=2G
|
||||
CPUQuota=200%
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
28
deploy/debian/systemd/rss2-qdrant-worker.service
Normal file
28
deploy/debian/systemd/rss2-qdrant-worker.service
Normal file
|
|
@ -0,0 +1,28 @@
|
|||
[Unit]
|
||||
Description=RSS2 Qdrant Sync Worker (Go)
|
||||
After=network.target postgresql.service rss2-qdrant.service
|
||||
Requires=postgresql.service rss2-qdrant.service
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=rss2
|
||||
Group=rss2
|
||||
WorkingDirectory=/opt/rss2
|
||||
EnvironmentFile=/opt/rss2/.env
|
||||
Environment=QDRANT_HOST=127.0.0.1
|
||||
Environment=QDRANT_PORT=6333
|
||||
Environment=QDRANT_COLLECTION=news_vectors
|
||||
Environment=QDRANT_SLEEP=30
|
||||
Environment=QDRANT_BATCH=100
|
||||
ExecStart=/opt/rss2/bin/qdrant_worker
|
||||
Restart=always
|
||||
RestartSec=10
|
||||
StandardOutput=journal
|
||||
StandardError=journal
|
||||
SyslogIdentifier=rss2-qdrant-worker
|
||||
|
||||
MemoryMax=1G
|
||||
CPUQuota=100%
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
25
deploy/debian/systemd/rss2-qdrant.service
Normal file
25
deploy/debian/systemd/rss2-qdrant.service
Normal file
|
|
@ -0,0 +1,25 @@
|
|||
[Unit]
|
||||
Description=Qdrant Vector Database
|
||||
After=network.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=rss2
|
||||
Group=rss2
|
||||
WorkingDirectory=/opt/rss2/qdrant
|
||||
ExecStart=/opt/rss2/qdrant/qdrant
|
||||
Restart=always
|
||||
RestartSec=5
|
||||
StandardOutput=journal
|
||||
StandardError=journal
|
||||
SyslogIdentifier=rss2-qdrant
|
||||
|
||||
Environment=QDRANT__SERVICE__HTTP_PORT=6333
|
||||
Environment=QDRANT__SERVICE__GRPC_PORT=6334
|
||||
Environment=QDRANT__STORAGE__STORAGE_PATH=/opt/rss2/data/qdrant_storage
|
||||
|
||||
MemoryMax=4G
|
||||
CPUQuota=400%
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
26
deploy/debian/systemd/rss2-related.service
Normal file
26
deploy/debian/systemd/rss2-related.service
Normal file
|
|
@ -0,0 +1,26 @@
|
|||
[Unit]
|
||||
Description=RSS2 Related News Worker (Go)
|
||||
After=network.target postgresql.service
|
||||
Requires=postgresql.service
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=rss2
|
||||
Group=rss2
|
||||
WorkingDirectory=/opt/rss2
|
||||
EnvironmentFile=/opt/rss2/.env
|
||||
Environment=RELATED_SLEEP=10
|
||||
Environment=RELATED_BATCH=200
|
||||
Environment=RELATED_TOPK=10
|
||||
ExecStart=/opt/rss2/bin/related
|
||||
Restart=always
|
||||
RestartSec=10
|
||||
StandardOutput=journal
|
||||
StandardError=journal
|
||||
SyslogIdentifier=rss2-related
|
||||
|
||||
MemoryMax=1G
|
||||
CPUQuota=100%
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
25
deploy/debian/systemd/rss2-scraper.service
Normal file
25
deploy/debian/systemd/rss2-scraper.service
Normal file
|
|
@ -0,0 +1,25 @@
|
|||
[Unit]
|
||||
Description=RSS2 Scraper HTML (Go)
|
||||
After=network.target postgresql.service
|
||||
Requires=postgresql.service
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=rss2
|
||||
Group=rss2
|
||||
WorkingDirectory=/opt/rss2
|
||||
EnvironmentFile=/opt/rss2/.env
|
||||
Environment=SCRAPER_SLEEP=60
|
||||
Environment=SCRAPER_BATCH=10
|
||||
ExecStart=/opt/rss2/bin/scraper
|
||||
Restart=always
|
||||
RestartSec=10
|
||||
StandardOutput=journal
|
||||
StandardError=journal
|
||||
SyslogIdentifier=rss2-scraper
|
||||
|
||||
MemoryMax=512M
|
||||
CPUQuota=100%
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
25
deploy/debian/systemd/rss2-topics.service
Normal file
25
deploy/debian/systemd/rss2-topics.service
Normal file
|
|
@ -0,0 +1,25 @@
|
|||
[Unit]
|
||||
Description=RSS2 Topics Worker (Go)
|
||||
After=network.target postgresql.service
|
||||
Requires=postgresql.service
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=rss2
|
||||
Group=rss2
|
||||
WorkingDirectory=/opt/rss2
|
||||
EnvironmentFile=/opt/rss2/.env
|
||||
Environment=TOPICS_SLEEP=10
|
||||
Environment=TOPICS_BATCH=500
|
||||
ExecStart=/opt/rss2/bin/topics
|
||||
Restart=always
|
||||
RestartSec=10
|
||||
StandardOutput=journal
|
||||
StandardError=journal
|
||||
SyslogIdentifier=rss2-topics
|
||||
|
||||
MemoryMax=512M
|
||||
CPUQuota=100%
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
26
deploy/debian/systemd/rss2-translation-scheduler.service
Normal file
26
deploy/debian/systemd/rss2-translation-scheduler.service
Normal file
|
|
@ -0,0 +1,26 @@
|
|||
[Unit]
|
||||
Description=RSS2 Translation Scheduler (Python)
|
||||
After=network.target postgresql.service
|
||||
Requires=postgresql.service
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=rss2
|
||||
Group=rss2
|
||||
WorkingDirectory=/opt/rss2/src
|
||||
EnvironmentFile=/opt/rss2/.env
|
||||
Environment=TARGET_LANGS=es
|
||||
Environment=SCHEDULER_BATCH=1000
|
||||
Environment=SCHEDULER_SLEEP=30
|
||||
ExecStart=/opt/rss2/venv/bin/python -m workers.translation_scheduler
|
||||
Restart=always
|
||||
RestartSec=10
|
||||
StandardOutput=journal
|
||||
StandardError=journal
|
||||
SyslogIdentifier=rss2-translation-scheduler
|
||||
|
||||
MemoryMax=256M
|
||||
CPUQuota=50%
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
31
deploy/debian/systemd/rss2-translator.service
Normal file
31
deploy/debian/systemd/rss2-translator.service
Normal file
|
|
@ -0,0 +1,31 @@
|
|||
[Unit]
|
||||
Description=RSS2 Translator Worker NLLB-200 (Python)
|
||||
After=network.target postgresql.service rss2-translation-scheduler.service
|
||||
Requires=postgresql.service
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=rss2
|
||||
Group=rss2
|
||||
WorkingDirectory=/opt/rss2/src
|
||||
EnvironmentFile=/opt/rss2/.env
|
||||
Environment=TARGET_LANGS=es
|
||||
Environment=TRANSLATOR_BATCH=32
|
||||
Environment=CT2_MODEL_PATH=/opt/rss2/models/nllb-ct2
|
||||
Environment=CT2_DEVICE=cpu
|
||||
Environment=CT2_COMPUTE_TYPE=int8
|
||||
Environment=UNIVERSAL_MODEL=facebook/nllb-200-distilled-600M
|
||||
Environment=HF_HOME=/opt/rss2/hf_cache
|
||||
ExecStart=/opt/rss2/venv/bin/python -m workers.ctranslator_worker
|
||||
Restart=always
|
||||
RestartSec=15
|
||||
StandardOutput=journal
|
||||
StandardError=journal
|
||||
SyslogIdentifier=rss2-translator
|
||||
|
||||
# El modelo NLLB-200 consume bastante RAM en CPU
|
||||
MemoryMax=4G
|
||||
CPUQuota=200%
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
24
deploy/debian/systemd/rss2-wiki.service
Normal file
24
deploy/debian/systemd/rss2-wiki.service
Normal file
|
|
@ -0,0 +1,24 @@
|
|||
[Unit]
|
||||
Description=RSS2 Wiki Worker - imagenes Wikipedia (Go)
|
||||
After=network.target postgresql.service
|
||||
Requires=postgresql.service
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=rss2
|
||||
Group=rss2
|
||||
WorkingDirectory=/opt/rss2
|
||||
EnvironmentFile=/opt/rss2/.env
|
||||
Environment=WIKI_SLEEP=10
|
||||
ExecStart=/opt/rss2/bin/wiki_worker
|
||||
Restart=always
|
||||
RestartSec=10
|
||||
StandardOutput=journal
|
||||
StandardError=journal
|
||||
SyslogIdentifier=rss2-wiki
|
||||
|
||||
MemoryMax=256M
|
||||
CPUQuota=50%
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
Loading…
Add table
Add a link
Reference in a new issue