mejora en el modelo de traduccion

This commit is contained in:
jlimolina 2026-04-04 03:17:37 +02:00
parent be56de7dd4
commit ecd7a3cdf8
21 changed files with 3708 additions and 322 deletions

View file

@ -1,3 +1,16 @@
# ==================================================================================
# CONFIGURACIÓN DE SECRETOS - Se cargan desde .env
# ==================================================================================
# Las variables POSTGRES_PASSWORD, REDIS_PASSWORD, DB_PASS, SECRET_KEY, GRAFANA_PASSWORD
# deben estar definidas en un archivo .env antes de ejecutar docker compose up
#
# Para generar credenciales seguras automáticamente:
# ./generate_secure_credentials.sh --force
#
# Despliegue simple:
# docker compose up -d
# ==================================================================================
services:
db:
image: postgres:18-alpine
@ -25,6 +38,7 @@ services:
restart: unless-stopped
healthcheck:
test: [ "CMD-SHELL", "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB || exit 1" ]
# NOTA: Si falla, verifica que POSTGRES_PASSWORD esté definido en .env
interval: 5s
timeout: 5s
retries: 30
@ -43,7 +57,7 @@ services:
TZ: Europe/Madrid
# SEGURIDAD: Redis con autenticación
command: >
redis-server --appendonly yes --maxmemory 512mb --maxmemory-policy allkeys-lru --requirepass ${REDIS_PASSWORD}
redis-server --appendonly yes --maxmemory 512mb --maxmemory-policy allkeys-lru --requirepass ${REDIS_PASSWORD:-!ERROR!REDIS_PASSWORD_NEEDED!}
volumes:
- ./data/redis-data:/data
- /etc/timezone:/etc/timezone:ro
@ -55,6 +69,7 @@ services:
- rss2_redis
restart: unless-stopped
healthcheck:
# VALIDACIÓN: Si REDIS_PASSWORD está vacío, esto fallará y detendrá el despliegue
test: [ "CMD", "redis-cli", "--no-auth-warning", "-a", "${REDIS_PASSWORD}", "ping" ]
interval: 5s
timeout: 3s
@ -282,8 +297,6 @@ services:
dockerfile: Dockerfile.translator
image: rss2-translator:latest
command: bash -lc "python -m workers.ctranslator_worker"
security_opt:
- seccomp=unconfined
environment:
DB_HOST: db
DB_PORT: 5432
@ -299,14 +312,60 @@ services:
HF_HOME: /app/hf_cache
TZ: Europe/Madrid
TRANSLATOR_ID: ${TRANSLATOR_ID:-}
PYTORCH_ENABLE_MPS_FALLBACK: 1
volumes:
- ./workers:/app/workers
- ./hf_cache:/app/hf_cache
- ./models:/app/models
networks:
- backend
profiles:
- cpu-only
depends_on:
db:
condition: service_healthy
restart: unless-stopped
# ==================================================================================
# TRANSLATOR GPU - Multiple instances with --scale for parallel GPU processing
# Configure multiple GPU workers: docker compose up -d --scale translator-gpu=2
# ==================================================================================
translator-gpu:
build:
context: .
dockerfile: Dockerfile.translator-gpu
image: rss2-translator-gpu:latest
command: bash -lc "python -m workers.ctranslator_worker"
environment:
DB_HOST: db
DB_PORT: 5432
DB_NAME: ${DB_NAME:-rss}
DB_USER: ${DB_USER:-rss}
DB_PASS: ${DB_PASS}
TARGET_LANGS: es
TRANSLATOR_BATCH: 256
CT2_MODEL_PATH: /app/models/nllb-ct2-1.3b
CT2_DEVICE: cuda
CT2_COMPUTE_TYPE: float16
UNIVERSAL_MODEL: facebook/nllb-200-1.3B
HF_HOME: /app/hf_cache
TZ: Europe/Madrid
TRANSLATOR_ID: ${TRANSLATOR_ID:-}
PYTORCH_CUDA_ALLOC_CONF: max_split_size_mb:512
NCCL_DEBUG: INFO
volumes:
- ./workers:/app/workers
- ./hf_cache:/app/hf_cache
- ./models:/app/models
- /dev/nvidia:/dev/nvidia
- /usr/local/nvidia:/usr/local/nvidia:ro
networks:
- backend
deploy:
resources:
reservations:
devices:
- driver: nvidia
count: 1
capabilities: [gpu]
depends_on:
db:
condition: service_healthy
@ -335,8 +394,6 @@ services:
- ./models:/app/models
networks:
- backend
profiles:
- remote
deploy:
resources:
limits:
@ -381,52 +438,6 @@ services:
condition: service_healthy
restart: unless-stopped
# ==================================================================================
# TRANSLATOR GPU (CTranslate2 with CUDA)
# ==================================================================================
translator-gpu:
build:
context: .
dockerfile: Dockerfile.translator-gpu
image: rss2-translator-gpu:latest
container_name: rss2_translator_gpu
command: bash -lc "python -m workers.ctranslator_worker"
security_opt:
- seccomp=unconfined
environment:
DB_HOST: db
DB_PORT: 5432
DB_NAME: ${DB_NAME:-rss}
DB_USER: ${DB_USER:-rss}
DB_PASS: ${DB_PASS}
TARGET_LANGS: es
TRANSLATOR_BATCH: 64
CT2_MODEL_PATH: /app/models/nllb-ct2
CT2_DEVICE: cuda
CT2_COMPUTE_TYPE: float16
UNIVERSAL_MODEL: facebook/nllb-200-distilled-600M
HF_HOME: /app/hf_cache
TZ: Europe/Madrid
volumes:
- ./workers:/app/workers
- ./hf_cache:/app/hf_cache
- ./models:/app/models
networks:
- backend
deploy:
resources:
limits:
memory: 4G
reservations:
devices:
- driver: nvidia
count: 1
capabilities: [ gpu ]
depends_on:
db:
condition: service_healthy
restart: unless-stopped
embeddings:
build:
context: .