fix(backup): corregir descarga y añadir restauración de BD

- Backend usa DATABASE_URL (resolvePgConnInfo) en vez de DB_* que no
  existían en el contenedor API, arreglando el fallo de pg_dump
- Dockerfile backend: alpine 3.23 para pg_dump 18.4 (compatible con
  PostgreSQL 18 del servidor)
- BackupDatabase ya no usa docker exec (no disponible en la imagen)
- Nuevo endpoint POST /api/admin/restore para restaurar .sql o .zip vía psql
- UI de Configuración: sección Restaurar Base de Datos
- Regenerado go.sum para el build
This commit is contained in:
jlimolina 2026-08-10 15:25:41 +02:00
parent ecd7a3cdf8
commit a285021e6c
22 changed files with 7624 additions and 450 deletions

View file

@ -12,7 +12,7 @@ RUN go mod tidy
RUN CGO_ENABLED=0 GOOS=linux go build -buildvcs=false -o /server ./cmd/server
FROM alpine:3.19
FROM alpine:3.23
RUN apk add --no-cache ca-certificates tzdata postgresql-client

View file

@ -11,6 +11,7 @@ import (
"os"
"os/signal"
"strconv"
"strings"
"syscall"
"time"
@ -23,7 +24,6 @@ var (
logger *log.Logger
dbPool *pgxpool.Pool
qdrantURL string
ollamaURL string
collection = "news_vectors"
sleepSec = 30
batchSize = 100
@ -39,7 +39,6 @@ func loadConfig() {
qdrantHost := getEnv("QDRANT_HOST", "localhost")
qdrantPort := getEnvInt("QDRANT_PORT", 6333)
qdrantURL = fmt.Sprintf("http://%s:%d", qdrantHost, qdrantPort)
ollamaURL = getEnv("OLLAMA_URL", "http://ollama:11434")
collection = getEnv("QDRANT_COLLECTION", "news_vectors")
}
@ -61,7 +60,7 @@ func getEnvInt(key string, defaultValue int) int {
type Translation struct {
ID int64
NoticiaID int64
NoticiaID string
Lang string
Titulo string
Resumen string
@ -70,6 +69,7 @@ type Translation struct {
FuenteNombre string
CategoriaID *int64
PaisID *int64
Embedding []float64
}
func getPendingTranslations(ctx context.Context) ([]Translation, error) {
@ -84,9 +84,11 @@ func getPendingTranslations(ctx context.Context) ([]Translation, error) {
n.fecha,
n.fuente_nombre,
n.categoria_id,
n.pais_id
n.pais_id,
te.embedding::text
FROM traducciones t
INNER JOIN noticias n ON t.noticia_id = n.id
INNER JOIN traduccion_embeddings te ON te.traduccion_id = t.id
WHERE t.vectorized = FALSE
AND t.status = 'done'
ORDER BY t.created_at ASC
@ -100,54 +102,41 @@ func getPendingTranslations(ctx context.Context) ([]Translation, error) {
var translations []Translation
for rows.Next() {
var t Translation
var embText string
if err := rows.Scan(
&t.ID, &t.NoticiaID, &t.Lang, &t.Titulo, &t.Resumen,
&t.URL, &t.Fecha, &t.FuenteNombre, &t.CategoriaID, &t.PaisID,
&embText,
); err != nil {
logger.Printf("Error scanning translation row: %v", err)
continue
}
emb, err := parseEmbedding(embText)
if err != nil {
logger.Printf("Error parsing embedding for translation %d: %v", t.ID, err)
continue
}
t.Embedding = emb
translations = append(translations, t)
}
return translations, nil
}
type EmbeddingRequest struct {
Model string `json:"model"`
Input string `json:"input"`
}
type EmbeddingResponse struct {
Embedding []float64 `json:"embedding"`
}
func generateEmbedding(text string) ([]float64, error) {
reqBody := EmbeddingRequest{
Model: "mxbai-embed-large",
Input: text,
func parseEmbedding(s string) ([]float64, error) {
s = strings.Trim(strings.TrimSpace(s), "{}")
if s == "" {
return nil, fmt.Errorf("empty embedding")
}
body, err := json.Marshal(reqBody)
if err != nil {
return nil, err
parts := strings.Split(s, ",")
emb := make([]float64, len(parts))
for i, p := range parts {
v, err := strconv.ParseFloat(strings.TrimSpace(p), 64)
if err != nil {
return nil, err
}
emb[i] = v
}
client := &http.Client{Timeout: 60 * time.Second}
resp, err := client.Post(ollamaURL+"/api/embeddings", "application/json", bytes.NewReader(body))
if err != nil {
return nil, err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("Ollama returned status %d", resp.StatusCode)
}
var result EmbeddingResponse
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
return nil, err
}
return result.Embedding, nil
return emb, nil
}
type QdrantPoint struct {
@ -160,7 +149,7 @@ type QdrantUpsertRequest struct {
Points []QdrantPoint `json:"points"`
}
func ensureCollection() error {
func ensureCollection(ctx context.Context) error {
req, err := http.NewRequest("GET", qdrantURL+"/collections/"+collection, nil)
if err != nil {
return err
@ -177,16 +166,15 @@ func ensureCollection() error {
return nil
}
// Get embedding dimension
emb, err := generateEmbedding("test")
// Get embedding dimension from stored embeddings
var dimension int
err = dbPool.QueryRow(ctx, `SELECT dim FROM traduccion_embeddings LIMIT 1`).Scan(&dimension)
if err != nil {
return fmt.Errorf("failed to get embedding dimension: %w", err)
}
dimension := len(emb)
// Create collection
createReq := map[string]interface{}{
"name": collection,
"vectors": map[string]interface{}{
"size": dimension,
"distance": "Cosine",
@ -194,26 +182,35 @@ func ensureCollection() error {
}
body, _ := json.Marshal(createReq)
resp2, err := http.Post(qdrantURL+"/collections", "application/json", bytes.NewReader(body))
createURL := qdrantURL + "/collections/" + collection
req2, err := http.NewRequest(http.MethodPut, createURL, bytes.NewReader(body))
if err != nil {
return err
}
req2.Header.Set("Content-Type", "application/json")
resp2, err := http.DefaultClient.Do(req2)
if err != nil {
return err
}
defer resp2.Body.Close()
if resp2.StatusCode != http.StatusOK {
respBody, _ := io.ReadAll(resp2.Body)
return fmt.Errorf("Qdrant create collection returned %d: %s", resp2.StatusCode, string(respBody))
}
logger.Printf("Created collection %s with dimension %d", collection, dimension)
return nil
}
func uploadToQdrant(translations []Translation, embeddings [][]float64) error {
func uploadToQdrant(translations []Translation, pointIDs []string) error {
points := make([]QdrantPoint, 0, len(translations))
for i, t := range translations {
if embeddings[i] == nil {
if t.Embedding == nil || len(t.Embedding) == 0 {
continue
}
pointID := uuid.New().String()
payload := map[string]interface{}{
"news_id": t.NoticiaID,
"traduccion_id": t.ID,
@ -235,8 +232,8 @@ func uploadToQdrant(translations []Translation, embeddings [][]float64) error {
}
points = append(points, QdrantPoint{
ID: pointID,
Vector: embeddings[i],
ID: pointIDs[i],
Vector: t.Embedding,
Payload: payload,
})
}
@ -252,7 +249,12 @@ func uploadToQdrant(translations []Translation, embeddings [][]float64) error {
}
url := fmt.Sprintf("%s/collections/%s/points", qdrantURL, collection)
resp, err := http.Post(url, "application/json", bytes.NewReader(body))
req, err := http.NewRequest(http.MethodPut, url, bytes.NewReader(body))
if err != nil {
return err
}
req.Header.Set("Content-Type", "application/json")
resp, err := http.DefaultClient.Do(req)
if err != nil {
return err
}
@ -317,7 +319,7 @@ func main() {
ctx := context.Background()
if err := ensureCollection(); err != nil {
if err := ensureCollection(ctx); err != nil {
logger.Printf("Warning: Could not ensure collection: %v", err)
}
@ -330,8 +332,8 @@ func main() {
os.Exit(0)
}()
logger.Printf("Config: qdrant=%s, ollama=%s, collection=%s, sleep=%ds, batch=%d",
qdrantURL, ollamaURL, collection, sleepSec, batchSize)
logger.Printf("Config: qdrant=%s, collection=%s, sleep=%ds, batch=%d",
qdrantURL, collection, sleepSec, batchSize)
totalProcessed := 0
@ -351,30 +353,19 @@ func main() {
logger.Printf("Processing %d translations...", len(translations))
// Generate embeddings
embeddings := make([][]float64, len(translations))
for i, t := range translations {
text := fmt.Sprintf("%s %s", t.Titulo, t.Resumen)
emb, err := generateEmbedding(text)
if err != nil {
logger.Printf("Error generating embedding for %d: %v", t.ID, err)
continue
}
embeddings[i] = emb
}
// Upload to Qdrant
if err := uploadToQdrant(translations, embeddings); err != nil {
logger.Printf("Error uploading to Qdrant: %v", err)
continue
}
// Update DB status
// Generate point IDs once (used both in Qdrant and DB)
pointIDs := make([]string, len(translations))
for i := range translations {
pointIDs[i] = uuid.New().String()
}
// Upload to Qdrant
if err := uploadToQdrant(translations, pointIDs); err != nil {
logger.Printf("Error uploading to Qdrant: %v", err)
continue
}
// Update DB status
if err := updateTranslationStatus(ctx, translations, pointIDs); err != nil {
logger.Printf("Error updating status: %v", err)
}

View file

@ -187,6 +187,7 @@ func main() {
admin.POST("/entities/retype", handlers.PatchEntityTipo)
admin.GET("/backup", handlers.BackupDatabase)
admin.GET("/backup/news", handlers.BackupNewsZipped)
admin.POST("/restore", handlers.RestoreDatabase)
admin.GET("/users", handlers.GetUsers)
admin.POST("/users/:id/promote", handlers.PromoteUser)
admin.POST("/users/:id/demote", handlers.DemoteUser)

View file

@ -1,6 +1,8 @@
module github.com/rss2/backend
go 1.22
go 1.23
toolchain go1.23.12
require (
github.com/PuerkitoBio/goquery v1.9.2

158
backend/go.sum Normal file
View file

@ -0,0 +1,158 @@
github.com/PuerkitoBio/goquery v1.9.2 h1:4/wZksC3KgkQw7SQgkKotmKljk0M6V8TUvA8Wb4yPeE=
github.com/PuerkitoBio/goquery v1.9.2/go.mod h1:GHPCaP0ODyyxqcNoFGYlAprUFH81NuRPd0GX3Zu2Mvk=
github.com/andybalholm/cascadia v1.3.2 h1:3Xi6Dw5lHF15JtdcmAHD3i1+T8plmv7BQ/nsViSLyss=
github.com/andybalholm/cascadia v1.3.2/go.mod h1:7gtRlve5FxPPgIgX36uWBX58OdBsSS6lUvCFb+h7KvU=
github.com/bsm/ginkgo/v2 v2.7.0 h1:ItPMPH90RbmZJt5GtkcNvIRuGEdwlBItdNVoyzaNQao=
github.com/bsm/ginkgo/v2 v2.7.0/go.mod h1:AiKlXPm7ItEHNc/2+OkrNG4E0ITzojb9/xWzvQ9XZ9w=
github.com/bsm/gomega v1.26.0 h1:LhQm+AFcgV2M0WyKroMASzAzCAJVpAxQXv4SaI9a69Y=
github.com/bsm/gomega v1.26.0/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0=
github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1O2AihPM=
github.com/bytedance/sonic v1.9.1 h1:6iJ6NqdoxCDr6mbY8h18oSO+cShGSMRGCEo7F2h0x8s=
github.com/bytedance/sonic v1.9.1/go.mod h1:i736AoUSYt75HyZLoJW9ERYxcy6eaN6h4BZXU064P/U=
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/chenzhuoyu/base64x v0.0.0-20211019084208-fb5309c8db06/go.mod h1:DH46F32mSOjUmXrMHnKwZdA8wcEefY7UVqBKYGjpdQY=
github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 h1:qSGYFH7+jGhDF8vLC+iwCD4WpbV1EBDSzWkJODFLams=
github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311/go.mod h1:b583jCggY9gE99b6G5LEC39OIiVsWj+R97kbl5odCEk=
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
github.com/gabriel-vasile/mimetype v1.4.2 h1:w5qFW6JKBz9Y393Y4q372O9A7cUSequkh1Q7OhCmWKU=
github.com/gabriel-vasile/mimetype v1.4.2/go.mod h1:zApsH/mKG4w07erKIaJPFiX0Tsq9BFQgN3qGY5GnNgA=
github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE=
github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI=
github.com/gin-gonic/gin v1.9.1 h1:4idEAncQnU5cB7BeOkPtxjfCSye0AAm1R0RVIqJ+Jmg=
github.com/gin-gonic/gin v1.9.1/go.mod h1:hPrL7YrpYKXt5YId3A/Tnip5kqbEAP+KLuI3SUcPTeU=
github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s=
github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA=
github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY=
github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY=
github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY=
github.com/go-playground/validator/v10 v10.14.0 h1:vgvQWe3XCz3gIeFDm/HnTIbj6UGmg/+t63MyGU2n5js=
github.com/go-playground/validator/v10 v10.14.0/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU=
github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU=
github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
github.com/golang-jwt/jwt/v5 v5.0.0 h1:1n1XNM9hk7O9mnQoNBGolZvzebBQ7p93ULHRc28XJUE=
github.com/golang-jwt/jwt/v5 v5.0.0/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk=
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/gorilla/websocket v1.5.1 h1:gmztn0JnHVt9JZquRuzLw3g4wouNVzKL15iLr/zn/QY=
github.com/gorilla/websocket v1.5.1/go.mod h1:x3kM2JMyaluk02fnUJpQuwD2dCS5NDG2ZHL0uE0tcaY=
github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk=
github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=
github.com/jackc/pgx/v5 v5.4.3 h1:cxFyXhxlvAifxnkKKdlxv8XqUf59tDlYjnV5YYfsJJY=
github.com/jackc/pgx/v5 v5.4.3/go.mod h1:Ig06C2Vu0t5qXC60W8sqIthScaEnFvojjj9dSljmHRA=
github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk=
github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
github.com/klauspost/cpuid/v2 v2.2.4 h1:acbojRNwl3o09bUq+yDCtZFc1aiwaAAxtcn8YkZXnvk=
github.com/klauspost/cpuid/v2 v2.2.4/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY=
github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0=
github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q=
github.com/leodido/go-urn v1.2.4/go.mod h1:7ZrI8mTSeBSHl/UaRyKQW1qZeMgak41ANeCNaVckg+4=
github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA=
github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
github.com/mmcdole/gofeed v1.2.1 h1:tPbFN+mfOLcM1kDF1x2c/N68ChbdBatkppdzf/vDe1s=
github.com/mmcdole/gofeed v1.2.1/go.mod h1:2wVInNpgmC85q16QTTuwbuKxtKkHLCDDtf0dCmnrNr4=
github.com/mmcdole/goxpp v1.1.0 h1:WwslZNF7KNAXTFuzRtn/OKZxFLJAAyOA9w82mDz2ZGI=
github.com/mmcdole/goxpp v1.1.0/go.mod h1:v+25+lT2ViuQ7mVxcncQ8ch1URund48oH+jhjiwEgS8=
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
github.com/pelletier/go-toml/v2 v2.0.8 h1:0ctb6s9mE31h0/lhu+J6OPmVeDxJn+kYnJc2jZR9tGQ=
github.com/pelletier/go-toml/v2 v2.0.8/go.mod h1:vuYfssBdrU2XDZ9bYydBu6t+6a6PYNcZljzZR9VXg+4=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/redis/go-redis/v9 v9.0.5 h1:CuQcn5HIEeK7BgElubPP8CGtE0KakrnbBSTLjathl5o=
github.com/redis/go-redis/v9 v9.0.5/go.mod h1:WqMKv5vnQbRuZstUwxQI195wHy+t4PuXDOjzMvcuQHk=
github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
github.com/stretchr/testify v1.8.3 h1:RP3t2pwF7cMEbC1dqtB6poj3niw/9gnV4Cjg5oW5gtY=
github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI=
github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08=
github.com/ugorji/go/codec v1.2.11 h1:BMaWp1Bb6fHwEtbplGBGJ498wD+LKlNSl25MjdZY4dU=
github.com/ugorji/go/codec v1.2.11/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg=
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8=
golang.org/x/arch v0.3.0 h1:02VY4/ZcO/gBOH6PUaoiptASxtXU10jazRCP865E97k=
golang.org/x/arch v0.3.0/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.26.0 h1:RrRspgV4mU+YwB4FYnuBoKsUapNIL5cohGAmSH3azsw=
golang.org/x/crypto v0.26.0/go.mod h1:GY7jblb9wI+FOo5y8/S2oY4zWP07AkOJ4+jxCqdqn54=
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns=
golang.org/x/net v0.28.0 h1:a9JDOJc5GMUJ0+UDqmLT86WiEy7iWyIhz8gz8E4e5hE=
golang.org/x/net v0.28.0/go.mod h1:yqtgsTWOOnlGLG9GFRrK3++bGOUEkNBoHZc8MEDWPNg=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ=
golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo=
golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
golang.org/x/text v0.17.0 h1:XtiM5bkSOt+ewxlOE/aE/AKEHibwj/6gvWMl9Rsh0Qc=
golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg=
google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4=

View file

@ -6,14 +6,15 @@ import (
"context"
"encoding/csv"
"fmt"
"io"
"log"
"net/http"
"net/url"
"os"
"os/exec"
"strconv"
"strings"
"time"
"github.com/gin-gonic/gin"
"github.com/rss2/backend/internal/config"
"github.com/rss2/backend/internal/db"
@ -637,17 +638,73 @@ func PatchEntityTipo(c *gin.Context) {
})
}
// pgConnInfo holds PostgreSQL connection parameters for pg_dump
type pgConnInfo struct {
host, port, name, user, pass string
}
// resolvePgConnInfo prefers DATABASE_URL (the variable the API container gets)
// and falls back to the individual DB_* variables used by the workers.
func resolvePgConnInfo() pgConnInfo {
if dbURL := os.Getenv("DATABASE_URL"); dbURL != "" {
if u, err := url.Parse(dbURL); err == nil && u.Hostname() != "" {
info := pgConnInfo{
host: u.Hostname(),
user: u.User.Username(),
name: strings.TrimPrefix(u.Path, "/"),
port: u.Port(),
}
if p, ok := u.User.Password(); ok {
info.pass = p
}
if info.port == "" {
info.port = "5432"
}
if info.user == "" {
info.user = "postgres"
}
if info.name == "" {
info.name = "postgres"
}
return info
}
}
info := pgConnInfo{
host: os.Getenv("DB_HOST"),
port: os.Getenv("DB_PORT"),
name: os.Getenv("DB_NAME"),
user: os.Getenv("DB_USER"),
pass: os.Getenv("DB_PASS"),
}
if info.host == "" {
info.host = "db"
}
if info.port == "" {
info.port = "5432"
}
if info.name == "" {
info.name = "rss"
}
if info.user == "" {
info.user = "rss"
}
return info
}
// BackupDatabase runs pg_dump and returns the SQL as a downloadable file
func BackupDatabase(c *gin.Context) {
// Ejecutar pg_dump desde dentro del contenedor db para evitar problemas de red
// Usamos docker exec para ejecutar el comando dentro del servicio db
cmd := exec.Command("docker", "exec", "rss2_db", "pg_dump",
"-U", os.Getenv("POSTGRES_USER"),
"-d", os.Getenv("POSTGRES_DB"),
info := resolvePgConnInfo()
cmd := exec.Command("pg_dump",
"-h", info.host,
"-p", info.port,
"-U", info.user,
"-d", info.name,
"--no-password",
"--format=plain",
"--pghost=5432",
)
cmd.Env = append(os.Environ(), fmt.Sprintf("PGPASSWORD=%s", info.pass))
var out bytes.Buffer
var stderr bytes.Buffer
@ -671,32 +728,16 @@ func BackupDatabase(c *gin.Context) {
// BackupNewsZipped performs a pg_dump of news tables and returns a ZIP file
func BackupNewsZipped(c *gin.Context) {
dbHost := os.Getenv("DB_HOST")
if dbHost == "" {
dbHost = "db"
}
dbPort := os.Getenv("DB_PORT")
if dbPort == "" {
dbPort = "5432"
}
dbName := os.Getenv("DB_NAME")
if dbName == "" {
dbName = "rss"
}
dbUser := os.Getenv("DB_USER")
if dbUser == "" {
dbUser = "rss"
}
dbPass := os.Getenv("DB_PASS")
info := resolvePgConnInfo()
// Tables to backup
tables := []string{"noticias", "traducciones", "tags", "tags_noticia"}
args := []string{
"-h", dbHost,
"-p", dbPort,
"-U", dbUser,
"-d", dbName,
"-h", info.host,
"-p", info.port,
"-U", info.user,
"-d", info.name,
"--no-password",
}
@ -705,7 +746,7 @@ func BackupNewsZipped(c *gin.Context) {
}
cmd := exec.Command("pg_dump", args...)
cmd.Env = append(os.Environ(), fmt.Sprintf("PGPASSWORD=%s", dbPass))
cmd.Env = append(os.Environ(), fmt.Sprintf("PGPASSWORD=%s", info.pass))
var sqlOut bytes.Buffer
var stderr bytes.Buffer
@ -748,3 +789,107 @@ func BackupNewsZipped(c *gin.Context) {
c.Header("Cache-Control", "no-cache")
c.Data(http.StatusOK, "application/zip", buf.Bytes())
}
// RestoreDatabase accepts an uploaded .sql or .zip backup and restores it via psql
func RestoreDatabase(c *gin.Context) {
file, err := c.FormFile("file")
if err != nil {
c.JSON(http.StatusBadRequest, gin.H{
"error": "No file uploaded",
"message": "Se requiere un archivo .sql o .zip",
})
return
}
uploaded, err := file.Open()
if err != nil {
c.JSON(http.StatusBadRequest, gin.H{"error": "Failed to open uploaded file"})
return
}
defer uploaded.Close()
data, err := io.ReadAll(uploaded)
if err != nil {
c.JSON(http.StatusBadRequest, gin.H{"error": "Failed to read uploaded file"})
return
}
var sqlData []byte
lowerName := strings.ToLower(file.Filename)
switch {
case strings.HasSuffix(lowerName, ".sql"):
sqlData = data
case strings.HasSuffix(lowerName, ".zip"):
zr, err := zip.NewReader(bytes.NewReader(data), int64(len(data)))
if err != nil {
c.JSON(http.StatusBadRequest, gin.H{"error": "Invalid ZIP file", "message": err.Error()})
return
}
found := false
for _, zf := range zr.File {
if zf.FileInfo().IsDir() || !strings.HasSuffix(strings.ToLower(zf.Name), ".sql") {
continue
}
rc, err := zf.Open()
if err != nil {
c.JSON(http.StatusBadRequest, gin.H{"error": "Failed to open SQL inside ZIP", "message": err.Error()})
return
}
sqlData, err = io.ReadAll(rc)
rc.Close()
if err != nil {
c.JSON(http.StatusBadRequest, gin.H{"error": "Failed to read SQL inside ZIP", "message": err.Error()})
return
}
found = true
break
}
if !found {
c.JSON(http.StatusBadRequest, gin.H{"error": "No SQL file found inside ZIP"})
return
}
default:
c.JSON(http.StatusBadRequest, gin.H{
"error": "Unsupported file type",
"message": "Solo se permiten archivos .sql o .zip",
})
return
}
if len(sqlData) == 0 {
c.JSON(http.StatusBadRequest, gin.H{"error": "Empty backup file"})
return
}
info := resolvePgConnInfo()
cmd := exec.Command("psql",
"-h", info.host,
"-p", info.port,
"-U", info.user,
"-d", info.name,
"--no-password",
"-v", "ON_ERROR_STOP=1",
)
cmd.Env = append(os.Environ(), fmt.Sprintf("PGPASSWORD=%s", info.pass))
cmd.Stdin = bytes.NewReader(sqlData)
var out bytes.Buffer
var stderr bytes.Buffer
cmd.Stdout = &out
cmd.Stderr = &stderr
if err := cmd.Run(); err != nil {
c.JSON(http.StatusInternalServerError, gin.H{
"error": "Restore failed",
"details": stderr.String(),
})
return
}
c.JSON(http.StatusOK, gin.H{
"message": "Base de datos restaurada correctamente",
"filename": file.Filename,
"output": out.String(),
})
}

View file

@ -6,6 +6,7 @@ import (
"fmt"
"io"
"net/http"
"net/url"
"strconv"
"strings"
@ -397,11 +398,46 @@ func ImportFeeds(c *gin.Context) {
}
reader := csv.NewReader(strings.NewReader(string(content)))
_, err = reader.Read()
records, err := reader.ReadAll()
if err != nil {
c.JSON(http.StatusBadRequest, models.ErrorResponse{Error: "Invalid CSV format"})
return
}
if len(records) == 0 {
c.JSON(http.StatusBadRequest, models.ErrorResponse{Error: "Empty CSV file"})
return
}
colIndex := func(header []string, names ...string) int {
for i, h := range header {
h = strings.ToLower(strings.TrimSpace(h))
for _, n := range names {
if h == n {
return i
}
}
}
return -1
}
header := records[0]
isHeader := colIndex(header, "url", "link", "feed", "nombre", "name", "titulo") != -1
var dataRows [][]string
iNombre, iDesc, iURL, iCat, iPais, iLang, iActivo, iFallos := -1, -1, -1, -1, -1, -1, -1, -1
if isHeader {
iNombre = colIndex(header, "nombre", "name", "titulo", "title")
iDesc = colIndex(header, "descripcion", "description")
iURL = colIndex(header, "url", "link", "feed")
iCat = colIndex(header, "categoria_id", "category_id", "categoria", "category")
iPais = colIndex(header, "pais_id", "country_id", "pais", "country")
iLang = colIndex(header, "idioma", "language", "lang")
iActivo = colIndex(header, "activo", "active", "enabled")
iFallos = colIndex(header, "fallos", "failures", "fails")
dataRows = records[1:]
} else {
dataRows = records
}
imported := 0
skipped := 0
@ -415,71 +451,115 @@ func ImportFeeds(c *gin.Context) {
}
defer tx.Rollback(context.Background())
for {
record, err := reader.Read()
if err == io.EOF {
break
}
if err != nil {
failed++
continue
field := func(record []string, idx int) string {
if idx < 0 || idx >= len(record) {
return ""
}
return strings.TrimSpace(record[idx])
}
if len(record) < 4 {
for _, record := range dataRows {
if len(record) == 0 {
skipped++
continue
}
nombre := strings.TrimSpace(record[1])
url := strings.TrimSpace(record[3])
var nombre, url string
if isHeader {
nombre = field(record, iNombre)
url = field(record, iURL)
} else if len(record) == 1 {
url = field(record, 0)
} else if len(record) == 2 {
a := field(record, 0)
b := field(record, 1)
if looksLikeURL(a) {
url, nombre = a, b
} else {
nombre, url = a, b
}
} else {
nombre = field(record, 1)
url = field(record, 3)
}
if nombre == "" || url == "" {
if url == "" {
skipped++
continue
}
if nombre == "" {
nombre = nombreFromURL(url)
}
var descripcion *string
if len(record) > 2 && strings.TrimSpace(record[2]) != "" {
descripcionStr := strings.TrimSpace(record[2])
descripcion = &descripcionStr
}
var categoriaID *int64
if len(record) > 4 && strings.TrimSpace(record[4]) != "" {
catID, err := strconv.ParseInt(strings.TrimSpace(record[4]), 10, 64)
if err == nil {
categoriaID = &catID
}
}
var paisID *int64
if len(record) > 6 && strings.TrimSpace(record[6]) != "" {
pID, err := strconv.ParseInt(strings.TrimSpace(record[6]), 10, 64)
if err == nil {
paisID = &pID
}
}
var idioma *string
if len(record) > 8 && strings.TrimSpace(record[8]) != "" {
lang := strings.TrimSpace(record[8])
if len(lang) > 2 {
lang = lang[:2]
}
idioma = &lang
}
activo := true
if len(record) > 9 && strings.TrimSpace(record[9]) != "" {
activo = strings.ToLower(strings.TrimSpace(record[9])) == "true"
var fallos int64
if isHeader {
if d := field(record, iDesc); d != "" {
descripcion = &d
}
if v := field(record, iCat); v != "" {
if id, err := strconv.ParseInt(v, 10, 64); err == nil {
categoriaID = &id
}
}
if v := field(record, iPais); v != "" {
if id, err := strconv.ParseInt(v, 10, 64); err == nil {
paisID = &id
}
}
if l := field(record, iLang); l != "" {
if len(l) > 2 {
l = l[:2]
}
idioma = &l
}
if a := field(record, iActivo); a != "" {
activo = strings.ToLower(a) == "true"
}
if f := field(record, iFallos); f != "" {
if v, err := strconv.ParseInt(f, 10, 64); err == nil {
fallos = v
}
}
} else {
if d := field(record, 2); d != "" {
descripcion = &d
}
if v := field(record, 4); v != "" {
if id, err := strconv.ParseInt(v, 10, 64); err == nil {
categoriaID = &id
}
}
if v := field(record, 6); v != "" {
if id, err := strconv.ParseInt(v, 10, 64); err == nil {
paisID = &id
}
}
if l := field(record, 8); l != "" {
if len(l) > 2 {
l = l[:2]
}
idioma = &l
}
if a := field(record, 9); a != "" {
activo = strings.ToLower(a) == "true"
}
if f := field(record, 10); f != "" {
if v, err := strconv.ParseInt(f, 10, 64); err == nil {
fallos = v
}
}
}
var fallos int64
if len(record) > 10 && strings.TrimSpace(record[10]) != "" {
f, err := strconv.ParseInt(strings.TrimSpace(record[10]), 10, 64)
if err == nil {
fallos = f
}
if _, err := tx.Exec(context.Background(), "SAVEPOINT sp"); err != nil {
failed++
errors = append(errors, fmt.Sprintf("Error for %s: %v", url, err))
continue
}
var existingID int64
@ -490,22 +570,25 @@ func ImportFeeds(c *gin.Context) {
WHERE id=$8`,
nombre, descripcion, categoriaID, paisID, idioma, activo, fallos, existingID,
)
if err != nil {
failed++
errors = append(errors, fmt.Sprintf("Error updating %s: %v", url, err))
continue
}
} else {
_, err = tx.Exec(context.Background(), `
INSERT INTO feeds (nombre, descripcion, url, categoria_id, pais_id, idioma, activo, fallos)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8)`,
nombre, descripcion, url, categoriaID, paisID, idioma, activo, fallos,
)
if err != nil {
failed++
errors = append(errors, fmt.Sprintf("Error inserting %s: %v", url, err))
continue
}
}
if err != nil {
tx.Exec(context.Background(), "ROLLBACK TO SAVEPOINT sp")
failed++
errors = append(errors, fmt.Sprintf("Error upserting %s: %v", url, err))
continue
}
if _, err := tx.Exec(context.Background(), "RELEASE SAVEPOINT sp"); err != nil {
failed++
errors = append(errors, fmt.Sprintf("Error for %s: %v", url, err))
continue
}
imported++
@ -525,6 +608,22 @@ func ImportFeeds(c *gin.Context) {
})
}
func looksLikeURL(s string) bool {
return strings.HasPrefix(s, "http://") || strings.HasPrefix(s, "https://") || strings.HasPrefix(s, "http://www.") || strings.HasPrefix(s, "https://www.") || strings.HasPrefix(s, "www.") || strings.HasPrefix(s, "feed://")
}
func nombreFromURL(u string) string {
u = strings.TrimSpace(u)
if !strings.HasPrefix(u, "http://") && !strings.HasPrefix(u, "https://") {
u = "https://" + u
}
parsed, err := url.Parse(u)
if err != nil || parsed.Host == "" {
return u
}
return strings.TrimPrefix(parsed.Host, "www.")
}
func stringOrEmpty(s *string) string {
if s == nil {
return ""

View file

@ -51,55 +51,45 @@ func SearchNews(c *gin.Context) {
offset := (page - 1) * perPage
// Build dynamic query
args := []interface{}{}
argNum := 1
args := []interface{}{lang}
argNum := 2
whereClause := "WHERE 1=1"
if query != "" {
whereClause += " AND (n.titulo ILIKE $" + strconv.Itoa(argNum) + " OR n.resumen ILIKE $" + strconv.Itoa(argNum) + " OR n.contenido ILIKE $" + strconv.Itoa(argNum) + ")"
whereClause += " AND (n.titulo ILIKE $" + strconv.Itoa(argNum) + " OR n.resumen ILIKE $" + strconv.Itoa(argNum) + ")"
args = append(args, "%"+query+"%")
argNum++
}
if lang != "" {
whereClause += " AND t.lang_to = $" + strconv.Itoa(argNum)
args = append(args, lang)
argNum++
}
if categoriaID != "" {
whereClause += " AND n.categoria_id = $" + strconv.Itoa(argNum)
catID, err := strconv.ParseInt(categoriaID, 10, 64)
if err == nil {
if catID, err := strconv.ParseInt(categoriaID, 10, 64); err == nil {
whereClause += " AND n.categoria_id = $" + strconv.Itoa(argNum)
args = append(args, catID)
argNum++
}
}
if paisID != "" {
whereClause += " AND n.pais_id = $" + strconv.Itoa(argNum)
pID, err := strconv.ParseInt(paisID, 10, 64)
if err == nil {
if pID, err := strconv.ParseInt(paisID, 10, 64); err == nil {
whereClause += " AND n.pais_id = $" + strconv.Itoa(argNum)
args = append(args, pID)
argNum++
}
}
args = append(args, perPage, offset)
sqlQuery := `
SELECT n.id, n.titulo, n.resumen, n.contenido, n.url, n.imagen,
n.feed_id, n.lang, n.categoria_id, n.pais_id, n.created_at, n.updated_at,
COALESCE(t.titulo_trad, '') as titulo_trad,
COALESCE(t.resumen_trad, '') as resumen_trad,
t.lang_to as lang_trad,
f.nombre as fuente_nombre
SELECT n.id, n.titulo, COALESCE(n.resumen, ''), n.url, n.fecha, n.imagen_url,
n.categoria_id, n.pais_id, n.fuente_nombre,
t.titulo_trad,
t.resumen_trad,
t.lang_to as lang_trad
FROM noticias n
LEFT JOIN traducciones t ON t.noticia_id = n.id AND t.lang_to = $` + strconv.Itoa(argNum) + `
LEFT JOIN feeds f ON f.id = n.feed_id
LEFT JOIN traducciones t ON t.noticia_id = n.id AND t.lang_to = $1
` + whereClause + `
ORDER BY n.created_at DESC
LIMIT $` + strconv.Itoa(argNum+1) + ` OFFSET $` + strconv.Itoa(argNum+2)
ORDER BY n.fecha DESC
LIMIT $` + strconv.Itoa(argNum) + ` OFFSET $` + strconv.Itoa(argNum+1)
args = append(args, perPage, offset)
rows, err := db.GetPool().Query(ctx, sqlQuery, args...)
if err != nil {
@ -108,33 +98,38 @@ func SearchNews(c *gin.Context) {
}
defer rows.Close()
var newsList []models.NewsWithTranslations
var newsList []NewsResponse
for rows.Next() {
var n models.NewsWithTranslations
var imagen *string
var n NewsResponse
var imagenURL, fuenteNombre *string
var categoriaIDp, paisIDp *int64
err := rows.Scan(
&n.ID, &n.Title, &n.Summary, &n.Content, &n.URL, &imagen,
&n.FeedID, &n.Lang, &n.CategoryID, &n.CountryID, &n.CreatedAt, &n.UpdatedAt,
&n.ID, &n.Titulo, &n.Resumen, &n.URL, &n.Fecha, &imagenURL,
&categoriaIDp, &paisIDp, &fuenteNombre,
&n.TitleTranslated, &n.SummaryTranslated, &n.LangTranslated,
)
if err != nil {
continue
}
if imagen != nil {
n.ImageURL = imagen
if imagenURL != nil {
n.ImagenURL = imagenURL
}
if fuenteNombre != nil {
n.FuenteNombre = *fuenteNombre
}
n.CategoriaID = categoriaIDp
n.PaisID = paisIDp
newsList = append(newsList, n)
}
// Get total count
countArgs := args[:len(args)-2]
// Remove LIMIT/OFFSET from args for count
var total int
err = db.GetPool().QueryRow(ctx, `
SELECT COUNT(*) FROM noticias n
LEFT JOIN traducciones t ON t.noticia_id = n.id AND t.lang_to = $`+strconv.Itoa(argNum)+`
LEFT JOIN traducciones t ON t.noticia_id = n.id AND t.lang_to = $1
`+whereClause, countArgs...).Scan(&total)
if err != nil {
total = len(newsList)
@ -142,15 +137,13 @@ func SearchNews(c *gin.Context) {
totalPages := (total + perPage - 1) / perPage
response := models.NewsListResponse{
News: newsList,
Total: total,
Page: page,
PerPage: perPage,
TotalPages: totalPages,
}
c.JSON(http.StatusOK, response)
c.JSON(http.StatusOK, gin.H{
"news": newsList,
"total": total,
"page": page,
"per_page": perPage,
"total_pages": totalPages,
})
}
func GetStats(c *gin.Context) {

View file

@ -132,7 +132,10 @@ func writeLoop(wsWorker *WSWorker) {
for {
select {
case <-ticker.C:
if err := wsWorker.conn.WriteMessage(websocket.PingMessage, nil); err != nil {
wsWorker.mu.Lock()
err := wsWorker.conn.WriteMessage(websocket.PingMessage, nil)
wsWorker.mu.Unlock()
if err != nil {
return
}
}
@ -163,7 +166,9 @@ func heartbeatLoop(wsWorker *WSWorker) {
func sendWS(wsWorker *WSWorker, msg models.WSServerMessage) {
data, _ := json.Marshal(msg)
wsWorker.mu.Lock()
wsWorker.conn.WriteMessage(websocket.TextMessage, data)
wsWorker.mu.Unlock()
}
func cleanupWorker(wsWorker *WSWorker, apiKey string) {

16
backup.sh Executable file
View file

@ -0,0 +1,16 @@
#!/bin/bash
# Backup diario de PostgreSQL rss2
set -e
BACKUP_DIR="/home/x/rss2/backups"
TIMESTAMP=$(date +%Y%m%d_%H%M%S)
OUT="$BACKUP_DIR/rss-${TIMESTAMP}.sql.gz"
LOG="$BACKUP_DIR/backup.log"
mkdir -p "$BACKUP_DIR"
docker exec rss2_db pg_dump -U rss -d rss --no-owner --no-privileges | gzip > "$OUT"
# Retener solo los ultimos 30 dias (borrado seguro de ficheros concretos, nunca rm -rf)
find "$BACKUP_DIR" -maxdepth 1 -name 'rss-*.sql.gz' -mtime +30 -delete
echo "$(date '+%F %T') Backup OK: $OUT ($(du -h "$OUT" | cut -f1))" >> "$LOG"

10
docker-compose.local.yml Normal file
View file

@ -0,0 +1,10 @@
# Override local: permite acceder a Grafana en http://localhost:3001
# La red monitoring es internal:true y compose descarta los puertos de servicios
# que solo están en redes internas. Añadir la red frontend (no internal) permite
# publicar el puerto 127.0.0.1:3001.
# Uso: docker compose -f docker-compose.yml -f docker-compose.local.yml up -d
services:
grafana:
networks:
- monitoring
- frontend

View file

@ -308,6 +308,7 @@ services:
CT2_MODEL_PATH: /app/models/nllb-ct2
CT2_DEVICE: cpu
CT2_COMPUTE_TYPE: int8
CT2_INTRA_THREADS: 4
UNIVERSAL_MODEL: facebook/nllb-200-distilled-600M
HF_HOME: /app/hf_cache
TZ: Europe/Madrid
@ -325,14 +326,13 @@ services:
restart: unless-stopped
# ==================================================================================
# TRANSLATOR GPU - Multiple instances with --scale for parallel GPU processing
# Configure multiple GPU workers: docker compose up -d --scale translator-gpu=2
# TRANSLATOR CPU Nº2 - Segundo worker CPU (paralelo al translator)
# ==================================================================================
translator-gpu:
build:
context: .
dockerfile: Dockerfile.translator-gpu
image: rss2-translator-gpu:latest
dockerfile: Dockerfile.translator
image: rss2-translator:latest
command: bash -lc "python -m workers.ctranslator_worker"
environment:
DB_HOST: db
@ -341,70 +341,27 @@ services:
DB_USER: ${DB_USER:-rss}
DB_PASS: ${DB_PASS}
TARGET_LANGS: es
TRANSLATOR_BATCH: 256
CT2_MODEL_PATH: /app/models/nllb-ct2-1.3b
CT2_DEVICE: cuda
CT2_COMPUTE_TYPE: float16
UNIVERSAL_MODEL: facebook/nllb-200-1.3B
TRANSLATOR_BATCH: 32
CT2_MODEL_PATH: /app/models/nllb-ct2
CT2_DEVICE: cpu
CT2_COMPUTE_TYPE: int8
CT2_INTRA_THREADS: 4
UNIVERSAL_MODEL: facebook/nllb-200-distilled-600M
HF_HOME: /app/hf_cache
TZ: Europe/Madrid
TRANSLATOR_ID: ${TRANSLATOR_ID:-}
PYTORCH_CUDA_ALLOC_CONF: max_split_size_mb:512
NCCL_DEBUG: INFO
PYTORCH_ENABLE_MPS_FALLBACK: 1
volumes:
- ./workers:/app/workers
- ./hf_cache:/app/hf_cache
- ./models:/app/models
- /dev/nvidia:/dev/nvidia
- /usr/local/nvidia:/usr/local/nvidia:ro
networks:
- backend
deploy:
resources:
reservations:
devices:
- driver: nvidia
count: 1
capabilities: [gpu]
depends_on:
db:
condition: service_healthy
restart: unless-stopped
# ==================================================================================
# REMOTE TRANSLATOR WORKER - Worker remoto que se conecta por WebSocket
# ==================================================================================
remote-translator:
build:
context: .
dockerfile: Dockerfile.remote-worker
container_name: rss2_remote_translator
environment:
WORKER_NAME: ${WORKER_NAME:-remote-worker-1}
WORKER_API_KEY: ${WORKER_API_KEY:-}
WORKER_SERVER: ${WORKER_SERVER:-ws://backend-go:8080/ws/worker}
CT2_DEVICE: ${CT2_DEVICE:-cuda}
CT2_MODEL_PATH: /app/models/nllb-ct2
CT2_COMPUTE_TYPE: ${CT2_COMPUTE_TYPE:-float16}
UNIVERSAL_MODEL: facebook/nllb-200-1.3B
HF_HOME: /app/hf_cache
TZ: Europe/Madrid
volumes:
- ./hf_cache:/app/hf_cache
- ./models:/app/models
networks:
- backend
deploy:
resources:
limits:
memory: 6G
reservations:
devices:
- driver: nvidia
count: 1
capabilities: [ gpu ]
restart: unless-stopped
# ==================================================================================
# TRANSLATION SCHEDULER - Creates translation jobs
# ==================================================================================
@ -455,7 +412,7 @@ services:
EMB_SLEEP_IDLE: 5
EMB_LANGS: es
EMB_LIMIT: 1000
DEVICE: cuda
DEVICE: cpu
HF_HOME: /app/hf_cache
TZ: Europe/Madrid
volumes:
@ -467,11 +424,6 @@ services:
resources:
limits:
memory: 6G
reservations:
devices:
- driver: nvidia
count: 1
capabilities: [ gpu ]
depends_on:
db:
condition: service_healthy
@ -523,7 +475,7 @@ services:
RELATED_SLEEP: 10
RELATED_BATCH: 200
RELATED_TOPK: 10
EMB_MODEL: mxbai-embed-large
EMB_MODEL: sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2
TZ: Europe/Madrid
networks:
- backend
@ -660,117 +612,8 @@ services:
memory: 2G
# ==================================================================================
# LLM CATEGORIZER (Python) - Categorización con Ollama
# REDES SEGMENTADAS
# ==================================================================================
llm-categorizer:
build:
context: .
dockerfile: Dockerfile
container_name: rss2_llm_categorizer
command: bash -lc "python -m workers.simple_categorizer_worker"
environment:
DB_HOST: db
DB_PORT: 5432
DB_NAME: ${DB_NAME:-rss}
DB_USER: ${DB_USER:-rss}
DB_PASS: ${DB_PASS}
CATEGORIZER_BATCH_SIZE: 10
CATEGORIZER_SLEEP_IDLE: 5
TZ: Europe/Madrid
volumes:
- ./workers:/app/workers
networks:
- backend
depends_on:
db:
condition: service_healthy
restart: unless-stopped
deploy:
resources:
limits:
cpus: '2'
memory: 1G
# ==================================================================================
# MONITORING STACK - SECURED
# ==================================================================================
prometheus:
image: prom/prometheus:latest
container_name: rss2_prometheus
volumes:
- ./monitoring/prometheus.yml:/etc/prometheus/prometheus.yml:ro
- prometheus_data:/prometheus
command:
- '--config.file=/etc/prometheus/prometheus.yml'
- '--storage.tsdb.path=/prometheus'
- '--web.console.libraries=/usr/share/prometheus/console_libraries'
- '--web.console.templates=/usr/share/prometheus/consoles'
# SEGURIDAD: Sin exposición de puertos - acceso solo vía Grafana o túnel SSH
# ports:
# - "9090:9090"
networks:
- monitoring
restart: unless-stopped
deploy:
resources:
limits:
cpus: '1'
memory: 2G
grafana:
image: grafana/grafana:latest
container_name: rss2_grafana
# SEGURIDAD: Acceso solo en localhost o vía túnel SSH
# Para acceso remoto, usar túnel SSH: ssh -L 3001:localhost:3001 user@server
ports:
- "127.0.0.1:3001:3000"
environment:
# SEGURIDAD: Cambiar este password en producción
- GF_SECURITY_ADMIN_PASSWORD=${GRAFANA_PASSWORD:-change_this_password}
- GF_USERS_ALLOW_SIGN_UP=false
- GF_SERVER_ROOT_URL=http://localhost:3001
- GF_SECURITY_COOKIE_SECURE=false
- GF_SECURITY_COOKIE_SAMESITE=lax
volumes:
- grafana_data:/var/lib/grafana
networks:
- monitoring
depends_on:
- prometheus
restart: unless-stopped
deploy:
resources:
limits:
cpus: '1'
memory: 1G
cadvisor:
image: gcr.io/cadvisor/cadvisor:latest
container_name: rss2_cadvisor
# SEGURIDAD: Sin exposición de puertos - solo acceso interno
# ports:
# - "8081:8080"
volumes:
- /:/rootfs:ro
- /var/run:/var/run:ro
- /sys:/sys:ro
- /var/lib/docker/:/var/lib/docker:ro
- /dev/disk/:/dev/disk:ro
devices:
- /dev/kmsg
networks:
- monitoring
restart: unless-stopped
deploy:
resources:
limits:
cpus: '0.5'
memory: 512M
# ==================================================================================
# REDES SEGMENTADAS
# ==================================================================================
networks:
# Red frontal - Solo nginx y web app
frontend:
@ -784,13 +627,5 @@ networks:
driver: bridge
internal: false # Acceso externo permitido (necesario para ingestor)
# Red de monitoreo - Prometheus, Grafana, cAdvisor
monitoring:
name: rss2_monitoring
driver: bridge
internal: true
volumes:
prometheus_data:
grafana_data:
torch_extensions:

View file

@ -1,27 +1,23 @@
#!/bin/bash
set -e
# Detectar si la base de datos necesita reinicialización
# Directorio de datos de PostgreSQL
PGDATA_DIR="/var/lib/postgresql/data/18/main"
echo "RSS2: Checking database integrity..."
echo "RSS2: Checking database presence..."
# Si no existe el archivo de versión, es una base de datos nueva
if [ ! -f "$PGDATA_DIR/PG_VERSION" ]; then
echo "RSS2: New database - will be initialized by docker-entrypoint"
# REGLA: nunca se borra ni se modifica el directorio de datos automáticamente.
# Solo se informa del estado y se deja que el entrypoint oficial de PostgreSQL
# inicialice (si no existe) o arranque con los datos existentes.
if [ -f "$PGDATA_DIR/PG_VERSION" ]; then
echo "RSS2: Existing database found at $PGDATA_DIR - starting normally"
else
# Verificar si la base de datos es funcional
if ! pg_isready -h localhost -p 5432 -U "${POSTGRES_USER:-rss}" 2>/dev/null; then
echo "RSS2: Database appears corrupted - removing old data files for fresh initialization..."
# Eliminar solo los archivos de datos, no todo el directorio
rm -rf "$PGDATA_DIR"/*
echo "RSS2: Data files removed - docker-entrypoint will initialize fresh database"
else
echo "RSS2: Database is healthy"
fi
echo "RSS2: No database found at $PGDATA_DIR - docker-entrypoint will initialize it"
fi
# Ejecutar el entrypoint original con los parámetros de PostgreSQL
# Ejecutar el entrypoint original con los parámetros de PostgreSQL.
# Si la base de datos estuviera corrupta, PostgreSQL fallará al arrancar y
# registrará el error en los logs; NO se elimina ningún dato de forma automática.
exec docker-entrypoint.sh \
postgres \
-c max_connections=200 \

6822
feeds.csv

File diff suppressed because it is too large Load diff

View file

@ -1,9 +1,11 @@
import { useState } from 'react'
import { apiService } from '../services/api'
import { AlertTriangle, Database, RefreshCw, Download, FileArchive } from 'lucide-react'
import { AlertTriangle, Database, RefreshCw, Download, FileArchive, Upload } from 'lucide-react'
export function AdminSettings() {
const [resetting, setResetting] = useState(false)
const [restoring, setRestoring] = useState(false)
const [restoreFile, setRestoreFile] = useState<File | null>(null)
const [message, setMessage] = useState<{ type: 'success' | 'error'; text: string } | null>(null)
const handleReset = async () => {
@ -35,6 +37,31 @@ export function AdminSettings() {
}
}
const handleRestore = async () => {
if (!restoreFile) {
setMessage({ type: 'error', text: 'Selecciona un archivo de copia de seguridad (.sql o .zip)' })
return
}
if (!confirm('¿Restaurar la base de datos?\n\nEsta acción sobrescribirá los datos existentes con el contenido del archivo de copia de seguridad. No se puede deshacer.')) return
setRestoring(true)
setMessage(null)
try {
const result = await apiService.restoreDatabase(restoreFile)
setMessage({ type: 'success', text: result.message })
setRestoreFile(null)
} catch (err: any) {
setMessage({
type: 'error',
text: err?.response?.data?.details || err?.response?.data?.error || 'Error al restaurar la base de datos',
})
} finally {
setRestoring(false)
}
}
return (
<div className="p-6">
<h1 className="text-2xl font-bold mb-6 flex items-center gap-2">
@ -42,6 +69,16 @@ export function AdminSettings() {
Configuración del Sistema
</h1>
{message && (
<div className={`max-w-2xl mb-6 p-3 rounded-md text-sm ${
message.type === 'success'
? 'bg-green-100 text-green-800 dark:bg-green-900/30 dark:text-green-200'
: 'bg-red-100 text-red-800 dark:bg-red-900/30 dark:text-red-200'
}`}>
{message.text}
</div>
)}
<div className="max-w-2xl space-y-6">
<div className="bg-white dark:bg-gray-800 rounded-lg shadow p-6">
<h2 className="text-lg font-semibold mb-4 flex items-center gap-2">
@ -75,16 +112,6 @@ export function AdminSettings() {
</>
)}
</button>
{message && (
<div className={`mt-4 p-3 rounded-md text-sm ${
message.type === 'success'
? 'bg-green-100 text-green-800 dark:bg-green-900/30 dark:text-green-200'
: 'bg-red-100 text-red-800 dark:bg-red-900/30 dark:text-red-200'
}`}>
{message.text}
</div>
)}
</div>
</div>
@ -112,6 +139,47 @@ export function AdminSettings() {
Descargar Backup .zip
</button>
</div>
<div className="p-4 border border-gray-100 dark:border-gray-700 rounded-lg bg-gray-50 dark:bg-gray-900/50">
<h3 className="font-medium mb-1 flex items-center gap-2">
<Upload className="h-4 w-4 text-blue-600" />
Restaurar Base de Datos
</h3>
<p className="text-sm text-gray-500 mb-4">
Sube un archivo de copia de seguridad (.sql o .zip) para restaurar la base de datos.
La restauración sobrescribe los datos existentes. Se recomienda restaurar después de un reset.
</p>
<div className="flex items-center gap-3 flex-wrap">
<input
type="file"
accept=".sql,.zip"
onChange={(e) => setRestoreFile(e.target.files?.[0] ?? null)}
className="block w-full max-w-sm text-sm text-gray-500 dark:text-gray-400 file:mr-4 file:py-2 file:px-4 file:rounded-md file:border-0 file:bg-blue-600 file:text-white file:cursor-pointer hover:file:bg-blue-700"
/>
<button
onClick={handleRestore}
disabled={restoring || !restoreFile}
className="flex items-center gap-2 px-4 py-2 bg-blue-600 text-white rounded-md hover:bg-blue-700 disabled:opacity-50 disabled:cursor-not-allowed"
>
{restoring ? (
<>
<RefreshCw className="h-4 w-4 animate-spin" />
Restaurando...
</>
) : (
<>
<Upload className="h-4 w-4" />
Restaurar Base de Datos
</>
)}
</button>
</div>
{restoreFile && (
<p className="mt-2 text-sm text-gray-500 dark:text-gray-400">
Archivo seleccionado: {restoreFile.name}
</p>
)}
</div>
</div>
</div>

View file

@ -132,9 +132,7 @@ export const apiService = {
importFeeds: async (file: File): Promise<{ imported: number; skipped: number; failed: number; message: string }> => {
const formData = new FormData()
formData.append('file', file)
const { data } = await api.post('/feeds/import', formData, {
headers: { 'Content-Type': 'multipart/form-data' },
})
const { data } = await api.post('/feeds/import', formData)
return data
},
@ -192,4 +190,11 @@ export const apiService = {
alert('Error al descargar la copia de seguridad. Verifica tu conexión o permisos.');
}
},
restoreDatabase: async (file: File): Promise<{ message: string; filename: string; output?: string }> => {
const formData = new FormData()
formData.append('file', file)
const { data } = await api.post('/admin/restore', formData)
return data
},
}

View file

@ -0,0 +1,11 @@
BEGIN;
-- Columnas para el clustering de noticias (cluster_worker.py)
ALTER TABLE eventos
ADD COLUMN IF NOT EXISTS centroid JSONB,
ADD COLUMN IF NOT EXISTS total_traducciones INTEGER DEFAULT 1,
ADD COLUMN IF NOT EXISTS fecha_inicio TIMESTAMP,
ADD COLUMN IF NOT EXISTS fecha_fin TIMESTAMP,
ADD COLUMN IF NOT EXISTS n_noticias INTEGER DEFAULT 1;
COMMIT;

View file

@ -0,0 +1,18 @@
BEGIN;
INSERT INTO topics (nombre, descripcion, weight, keywords) VALUES
('Política', 'Noticias políticas y de gobierno', 3, 'política,político,gobierno,presidente,ministro,parlamento,elecciones,partido,senado,diputado'),
('Economía', 'Economía y finanzas', 3, 'economía,económico,finanzas,mercado,inflación,banco,bancos,empresa,empresas,negocio,bolsa,inversión,presupuesto,impuestos'),
('Seguridad', 'Seguridad y defensa', 3, 'seguridad,ataque,ataques,explosión,terrorismo,terrorista,ejército,militar,patrulla,armas,policía'),
('Afganistán', 'Asuntos de Afganistán', 4, 'afganistán,afgano,afgana,kabul,candahar,kandahar,herat,mazar,talibán,talibanes,emirato'),
('Conflicto', 'Conflictos y guerra', 3, 'conflicto,guerra,enfrentamiento,enfrentamientos,crisis,víctimas,frontera,invasIón,ocupación'),
('Internacional', 'Relaciones internacionales', 2, 'internacional,diplomacia,diplomático,acuerdo,acuerdos,naciones unidas,embajador,tratado,extranjero'),
('Deportes', 'Deportes y competiciones', 2, 'deportes,deporte,fútbol,futbol,partido,equipo,liga,campeonato,tenis,baloncesto,cricket,selección'),
('Salud', 'Salud y medicina', 2, 'salud,hospital,médico,médicos,enfermedad,vacuna,tratamiento,pandemia,clínica,medicina'),
('Tecnología', 'Tecnología y digital', 2, 'tecnología,internet,digital,telefonía,telecomunicaciones,software,aplicación,aplicaciones,internet,tecnológico'),
('Educación', 'Educación y enseñanza', 2, 'educación,escuela,escuelas,universidad,universidades,estudiantes,profesor,enseñanza,academia,alumnos'),
('Cultura', 'Cultura y arte', 1, 'cultura,arte,literatura,cine,música,música,patrimonio,historia,museo,libro,libros'),
('Medio Ambiente', 'Clima y medio ambiente', 2, 'medio ambiente,clima,cambio climático,contaminación,sequía,inundación,agua,ecología,desastre natural')
ON CONFLICT (nombre) DO NOTHING;
COMMIT;

View file

@ -0,0 +1,7 @@
BEGIN;
-- El worker topics escribe en score (topics/main.go usa news_topics.score)
ALTER TABLE news_topics
ADD COLUMN IF NOT EXISTS score INTEGER DEFAULT 0;
COMMIT;

View file

@ -0,0 +1,10 @@
BEGIN;
-- El worker related (cmd/related/main.go) trabaja con traducciones
ALTER TABLE related_noticias
ADD COLUMN IF NOT EXISTS related_traduccion_id INTEGER REFERENCES traducciones(id) ON DELETE CASCADE;
CREATE UNIQUE INDEX IF NOT EXISTS idx_related_tr_pair
ON related_noticias(traduccion_id, related_traduccion_id);
COMMIT;

View file

@ -0,0 +1,2 @@
ALTER TABLE traducciones ADD COLUMN IF NOT EXISTS vectorization_date TIMESTAMP;
ALTER TABLE traducciones ADD COLUMN IF NOT EXISTS qdrant_point_id TEXT;

View file

@ -73,6 +73,8 @@ CT2_MODEL_PATH = _env_str("CT2_MODEL_PATH", "/app/models/nllb-ct2")
CT2_DEVICE = _env_str("CT2_DEVICE", "cpu")
CT2_COMPUTE_TYPE = _env_str("CT2_COMPUTE_TYPE", "int8")
UNIVERSAL_MODEL = _env_str("UNIVERSAL_MODEL", "facebook/nllb-200-distilled-600M")
CT2_INTRA_THREADS = _env_int("CT2_INTRA_THREADS", 0)
CT2_INTER_THREADS = _env_int("CT2_INTER_THREADS", 1)
BODY_CHARS_CHUNK = _env_int("BODY_CHARS_CHUNK", 900)
LANG_CODE_MAP = {
@ -128,7 +130,7 @@ def ensure_model():
all_files_ok = True
for f in required_files:
fpath = os.path.join(model_path, f)
if not os.path.exists(fpath) or os.path.getsize(fpath) < 1000:
if not os.path.exists(fpath) or os.path.getsize(fpath) < 100:
all_files_ok = False
break
@ -157,6 +159,8 @@ def ensure_model():
model_path,
device=device,
compute_type=CT2_COMPUTE_TYPE,
inter_threads=CT2_INTER_THREADS,
intra_threads=CT2_INTRA_THREADS,
)
_tokenizer = AutoTokenizer.from_pretrained(UNIVERSAL_MODEL)
@ -314,11 +318,7 @@ def translate_body_long(src: str, tgt: str, body: str) -> str:
if len(chunks) == 1:
return translate_texts(src, tgt, [body])[0]
translated_chunks = []
for ch in chunks:
tr = translate_texts(src, tgt, [ch])[0]
translated_chunks.append(tr)
translated_chunks = translate_texts(src, tgt, chunks)
return " ".join(translated_chunks)
@ -406,18 +406,34 @@ def process_batch(conn, rows):
titles = [i["titulo"] for i in items]
translated_titles = translate_texts(lang_from, lang_to, titles)
for item, tt in zip(items, translated_titles):
# Collect all body chunks across all items for a single batched call
flat_chunks = []
flat_keys = []
for item in items:
body = (item["resumen"] or "").strip()
tb = ""
if body:
try:
tb = translate_body_long(lang_from, lang_to, body)
except Exception as e:
LOG.error(f"Body translation error for ID {item['tr_id']}: {e}")
tb = item["resumen"]
chunks = split_body_into_chunks(body)
flat_chunks.extend(chunks)
flat_keys.extend([(item["tr_id"], i) for i in range(len(chunks))])
tt = clean_text((tt or "").strip())
tb = clean_text((tb or "").strip())
translated_bodies = []
if flat_chunks:
try:
translated_bodies = translate_texts(lang_from, lang_to, flat_chunks)
except Exception as e:
LOG.error(f"Batch body translation error: {e}")
translated_bodies = flat_chunks
body_parts = defaultdict(list)
for (tr_id, _), tr in zip(flat_keys, translated_bodies):
if tr is None:
continue
body_parts[tr_id].append(tr)
for idx, item in enumerate(items):
tt = clean_text((translated_titles[idx] or "").strip())
parts = body_parts.get(item["tr_id"])
tb = clean_text(" ".join(parts).strip()) if parts else ""
if not tt:
tt = item["titulo"]