mejora en el modelo de traduccion
This commit is contained in:
parent
be56de7dd4
commit
ecd7a3cdf8
21 changed files with 3708 additions and 322 deletions
|
|
@ -1,3 +1,16 @@
|
|||
# ==================================================================================
|
||||
# CONFIGURACIÓN DE SECRETOS - Se cargan desde .env
|
||||
# ==================================================================================
|
||||
# Las variables POSTGRES_PASSWORD, REDIS_PASSWORD, DB_PASS, SECRET_KEY, GRAFANA_PASSWORD
|
||||
# deben estar definidas en un archivo .env antes de ejecutar docker compose up
|
||||
#
|
||||
# Para generar credenciales seguras automáticamente:
|
||||
# ./generate_secure_credentials.sh --force
|
||||
#
|
||||
# Despliegue simple:
|
||||
# docker compose up -d
|
||||
# ==================================================================================
|
||||
|
||||
services:
|
||||
db:
|
||||
image: postgres:18-alpine
|
||||
|
|
@ -25,6 +38,7 @@ services:
|
|||
restart: unless-stopped
|
||||
healthcheck:
|
||||
test: [ "CMD-SHELL", "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB || exit 1" ]
|
||||
# NOTA: Si falla, verifica que POSTGRES_PASSWORD esté definido en .env
|
||||
interval: 5s
|
||||
timeout: 5s
|
||||
retries: 30
|
||||
|
|
@ -43,7 +57,7 @@ services:
|
|||
TZ: Europe/Madrid
|
||||
# SEGURIDAD: Redis con autenticación
|
||||
command: >
|
||||
redis-server --appendonly yes --maxmemory 512mb --maxmemory-policy allkeys-lru --requirepass ${REDIS_PASSWORD}
|
||||
redis-server --appendonly yes --maxmemory 512mb --maxmemory-policy allkeys-lru --requirepass ${REDIS_PASSWORD:-!ERROR!REDIS_PASSWORD_NEEDED!}
|
||||
volumes:
|
||||
- ./data/redis-data:/data
|
||||
- /etc/timezone:/etc/timezone:ro
|
||||
|
|
@ -55,6 +69,7 @@ services:
|
|||
- rss2_redis
|
||||
restart: unless-stopped
|
||||
healthcheck:
|
||||
# VALIDACIÓN: Si REDIS_PASSWORD está vacío, esto fallará y detendrá el despliegue
|
||||
test: [ "CMD", "redis-cli", "--no-auth-warning", "-a", "${REDIS_PASSWORD}", "ping" ]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
|
|
@ -282,8 +297,6 @@ services:
|
|||
dockerfile: Dockerfile.translator
|
||||
image: rss2-translator:latest
|
||||
command: bash -lc "python -m workers.ctranslator_worker"
|
||||
security_opt:
|
||||
- seccomp=unconfined
|
||||
environment:
|
||||
DB_HOST: db
|
||||
DB_PORT: 5432
|
||||
|
|
@ -299,14 +312,60 @@ services:
|
|||
HF_HOME: /app/hf_cache
|
||||
TZ: Europe/Madrid
|
||||
TRANSLATOR_ID: ${TRANSLATOR_ID:-}
|
||||
PYTORCH_ENABLE_MPS_FALLBACK: 1
|
||||
volumes:
|
||||
- ./workers:/app/workers
|
||||
- ./hf_cache:/app/hf_cache
|
||||
- ./models:/app/models
|
||||
networks:
|
||||
- backend
|
||||
profiles:
|
||||
- cpu-only
|
||||
depends_on:
|
||||
db:
|
||||
condition: service_healthy
|
||||
restart: unless-stopped
|
||||
|
||||
# ==================================================================================
|
||||
# TRANSLATOR GPU - Multiple instances with --scale for parallel GPU processing
|
||||
# Configure multiple GPU workers: docker compose up -d --scale translator-gpu=2
|
||||
# ==================================================================================
|
||||
translator-gpu:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile.translator-gpu
|
||||
image: rss2-translator-gpu:latest
|
||||
command: bash -lc "python -m workers.ctranslator_worker"
|
||||
environment:
|
||||
DB_HOST: db
|
||||
DB_PORT: 5432
|
||||
DB_NAME: ${DB_NAME:-rss}
|
||||
DB_USER: ${DB_USER:-rss}
|
||||
DB_PASS: ${DB_PASS}
|
||||
TARGET_LANGS: es
|
||||
TRANSLATOR_BATCH: 256
|
||||
CT2_MODEL_PATH: /app/models/nllb-ct2-1.3b
|
||||
CT2_DEVICE: cuda
|
||||
CT2_COMPUTE_TYPE: float16
|
||||
UNIVERSAL_MODEL: facebook/nllb-200-1.3B
|
||||
HF_HOME: /app/hf_cache
|
||||
TZ: Europe/Madrid
|
||||
TRANSLATOR_ID: ${TRANSLATOR_ID:-}
|
||||
PYTORCH_CUDA_ALLOC_CONF: max_split_size_mb:512
|
||||
NCCL_DEBUG: INFO
|
||||
volumes:
|
||||
- ./workers:/app/workers
|
||||
- ./hf_cache:/app/hf_cache
|
||||
- ./models:/app/models
|
||||
- /dev/nvidia:/dev/nvidia
|
||||
- /usr/local/nvidia:/usr/local/nvidia:ro
|
||||
networks:
|
||||
- backend
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: 1
|
||||
capabilities: [gpu]
|
||||
depends_on:
|
||||
db:
|
||||
condition: service_healthy
|
||||
|
|
@ -335,8 +394,6 @@ services:
|
|||
- ./models:/app/models
|
||||
networks:
|
||||
- backend
|
||||
profiles:
|
||||
- remote
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
|
|
@ -381,52 +438,6 @@ services:
|
|||
condition: service_healthy
|
||||
restart: unless-stopped
|
||||
|
||||
# ==================================================================================
|
||||
# TRANSLATOR GPU (CTranslate2 with CUDA)
|
||||
# ==================================================================================
|
||||
translator-gpu:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile.translator-gpu
|
||||
image: rss2-translator-gpu:latest
|
||||
container_name: rss2_translator_gpu
|
||||
command: bash -lc "python -m workers.ctranslator_worker"
|
||||
security_opt:
|
||||
- seccomp=unconfined
|
||||
environment:
|
||||
DB_HOST: db
|
||||
DB_PORT: 5432
|
||||
DB_NAME: ${DB_NAME:-rss}
|
||||
DB_USER: ${DB_USER:-rss}
|
||||
DB_PASS: ${DB_PASS}
|
||||
TARGET_LANGS: es
|
||||
TRANSLATOR_BATCH: 64
|
||||
CT2_MODEL_PATH: /app/models/nllb-ct2
|
||||
CT2_DEVICE: cuda
|
||||
CT2_COMPUTE_TYPE: float16
|
||||
UNIVERSAL_MODEL: facebook/nllb-200-distilled-600M
|
||||
HF_HOME: /app/hf_cache
|
||||
TZ: Europe/Madrid
|
||||
volumes:
|
||||
- ./workers:/app/workers
|
||||
- ./hf_cache:/app/hf_cache
|
||||
- ./models:/app/models
|
||||
networks:
|
||||
- backend
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
memory: 4G
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: 1
|
||||
capabilities: [ gpu ]
|
||||
depends_on:
|
||||
db:
|
||||
condition: service_healthy
|
||||
restart: unless-stopped
|
||||
|
||||
embeddings:
|
||||
build:
|
||||
context: .
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue