fix(backup): corregir descarga y añadir restauración de BD
- Backend usa DATABASE_URL (resolvePgConnInfo) en vez de DB_* que no existían en el contenedor API, arreglando el fallo de pg_dump - Dockerfile backend: alpine 3.23 para pg_dump 18.4 (compatible con PostgreSQL 18 del servidor) - BackupDatabase ya no usa docker exec (no disponible en la imagen) - Nuevo endpoint POST /api/admin/restore para restaurar .sql o .zip vía psql - UI de Configuración: sección Restaurar Base de Datos - Regenerado go.sum para el build
This commit is contained in:
parent
ecd7a3cdf8
commit
a285021e6c
22 changed files with 7624 additions and 450 deletions
|
|
@ -12,7 +12,7 @@ RUN go mod tidy
|
|||
|
||||
RUN CGO_ENABLED=0 GOOS=linux go build -buildvcs=false -o /server ./cmd/server
|
||||
|
||||
FROM alpine:3.19
|
||||
FROM alpine:3.23
|
||||
|
||||
RUN apk add --no-cache ca-certificates tzdata postgresql-client
|
||||
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@ import (
|
|||
"os"
|
||||
"os/signal"
|
||||
"strconv"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
|
|
@ -23,7 +24,6 @@ var (
|
|||
logger *log.Logger
|
||||
dbPool *pgxpool.Pool
|
||||
qdrantURL string
|
||||
ollamaURL string
|
||||
collection = "news_vectors"
|
||||
sleepSec = 30
|
||||
batchSize = 100
|
||||
|
|
@ -39,7 +39,6 @@ func loadConfig() {
|
|||
qdrantHost := getEnv("QDRANT_HOST", "localhost")
|
||||
qdrantPort := getEnvInt("QDRANT_PORT", 6333)
|
||||
qdrantURL = fmt.Sprintf("http://%s:%d", qdrantHost, qdrantPort)
|
||||
ollamaURL = getEnv("OLLAMA_URL", "http://ollama:11434")
|
||||
collection = getEnv("QDRANT_COLLECTION", "news_vectors")
|
||||
}
|
||||
|
||||
|
|
@ -61,7 +60,7 @@ func getEnvInt(key string, defaultValue int) int {
|
|||
|
||||
type Translation struct {
|
||||
ID int64
|
||||
NoticiaID int64
|
||||
NoticiaID string
|
||||
Lang string
|
||||
Titulo string
|
||||
Resumen string
|
||||
|
|
@ -70,6 +69,7 @@ type Translation struct {
|
|||
FuenteNombre string
|
||||
CategoriaID *int64
|
||||
PaisID *int64
|
||||
Embedding []float64
|
||||
}
|
||||
|
||||
func getPendingTranslations(ctx context.Context) ([]Translation, error) {
|
||||
|
|
@ -84,9 +84,11 @@ func getPendingTranslations(ctx context.Context) ([]Translation, error) {
|
|||
n.fecha,
|
||||
n.fuente_nombre,
|
||||
n.categoria_id,
|
||||
n.pais_id
|
||||
n.pais_id,
|
||||
te.embedding::text
|
||||
FROM traducciones t
|
||||
INNER JOIN noticias n ON t.noticia_id = n.id
|
||||
INNER JOIN traduccion_embeddings te ON te.traduccion_id = t.id
|
||||
WHERE t.vectorized = FALSE
|
||||
AND t.status = 'done'
|
||||
ORDER BY t.created_at ASC
|
||||
|
|
@ -100,54 +102,41 @@ func getPendingTranslations(ctx context.Context) ([]Translation, error) {
|
|||
var translations []Translation
|
||||
for rows.Next() {
|
||||
var t Translation
|
||||
var embText string
|
||||
if err := rows.Scan(
|
||||
&t.ID, &t.NoticiaID, &t.Lang, &t.Titulo, &t.Resumen,
|
||||
&t.URL, &t.Fecha, &t.FuenteNombre, &t.CategoriaID, &t.PaisID,
|
||||
&embText,
|
||||
); err != nil {
|
||||
logger.Printf("Error scanning translation row: %v", err)
|
||||
continue
|
||||
}
|
||||
emb, err := parseEmbedding(embText)
|
||||
if err != nil {
|
||||
logger.Printf("Error parsing embedding for translation %d: %v", t.ID, err)
|
||||
continue
|
||||
}
|
||||
t.Embedding = emb
|
||||
translations = append(translations, t)
|
||||
}
|
||||
return translations, nil
|
||||
}
|
||||
|
||||
type EmbeddingRequest struct {
|
||||
Model string `json:"model"`
|
||||
Input string `json:"input"`
|
||||
}
|
||||
|
||||
type EmbeddingResponse struct {
|
||||
Embedding []float64 `json:"embedding"`
|
||||
}
|
||||
|
||||
func generateEmbedding(text string) ([]float64, error) {
|
||||
reqBody := EmbeddingRequest{
|
||||
Model: "mxbai-embed-large",
|
||||
Input: text,
|
||||
func parseEmbedding(s string) ([]float64, error) {
|
||||
s = strings.Trim(strings.TrimSpace(s), "{}")
|
||||
if s == "" {
|
||||
return nil, fmt.Errorf("empty embedding")
|
||||
}
|
||||
|
||||
body, err := json.Marshal(reqBody)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
parts := strings.Split(s, ",")
|
||||
emb := make([]float64, len(parts))
|
||||
for i, p := range parts {
|
||||
v, err := strconv.ParseFloat(strings.TrimSpace(p), 64)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
emb[i] = v
|
||||
}
|
||||
|
||||
client := &http.Client{Timeout: 60 * time.Second}
|
||||
resp, err := client.Post(ollamaURL+"/api/embeddings", "application/json", bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, fmt.Errorf("Ollama returned status %d", resp.StatusCode)
|
||||
}
|
||||
|
||||
var result EmbeddingResponse
|
||||
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return result.Embedding, nil
|
||||
return emb, nil
|
||||
}
|
||||
|
||||
type QdrantPoint struct {
|
||||
|
|
@ -160,7 +149,7 @@ type QdrantUpsertRequest struct {
|
|||
Points []QdrantPoint `json:"points"`
|
||||
}
|
||||
|
||||
func ensureCollection() error {
|
||||
func ensureCollection(ctx context.Context) error {
|
||||
req, err := http.NewRequest("GET", qdrantURL+"/collections/"+collection, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
|
|
@ -177,16 +166,15 @@ func ensureCollection() error {
|
|||
return nil
|
||||
}
|
||||
|
||||
// Get embedding dimension
|
||||
emb, err := generateEmbedding("test")
|
||||
// Get embedding dimension from stored embeddings
|
||||
var dimension int
|
||||
err = dbPool.QueryRow(ctx, `SELECT dim FROM traduccion_embeddings LIMIT 1`).Scan(&dimension)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get embedding dimension: %w", err)
|
||||
}
|
||||
dimension := len(emb)
|
||||
|
||||
// Create collection
|
||||
createReq := map[string]interface{}{
|
||||
"name": collection,
|
||||
"vectors": map[string]interface{}{
|
||||
"size": dimension,
|
||||
"distance": "Cosine",
|
||||
|
|
@ -194,26 +182,35 @@ func ensureCollection() error {
|
|||
}
|
||||
|
||||
body, _ := json.Marshal(createReq)
|
||||
resp2, err := http.Post(qdrantURL+"/collections", "application/json", bytes.NewReader(body))
|
||||
createURL := qdrantURL + "/collections/" + collection
|
||||
req2, err := http.NewRequest(http.MethodPut, createURL, bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req2.Header.Set("Content-Type", "application/json")
|
||||
resp2, err := http.DefaultClient.Do(req2)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp2.Body.Close()
|
||||
|
||||
if resp2.StatusCode != http.StatusOK {
|
||||
respBody, _ := io.ReadAll(resp2.Body)
|
||||
return fmt.Errorf("Qdrant create collection returned %d: %s", resp2.StatusCode, string(respBody))
|
||||
}
|
||||
|
||||
logger.Printf("Created collection %s with dimension %d", collection, dimension)
|
||||
return nil
|
||||
}
|
||||
|
||||
func uploadToQdrant(translations []Translation, embeddings [][]float64) error {
|
||||
func uploadToQdrant(translations []Translation, pointIDs []string) error {
|
||||
points := make([]QdrantPoint, 0, len(translations))
|
||||
|
||||
for i, t := range translations {
|
||||
if embeddings[i] == nil {
|
||||
if t.Embedding == nil || len(t.Embedding) == 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
pointID := uuid.New().String()
|
||||
|
||||
payload := map[string]interface{}{
|
||||
"news_id": t.NoticiaID,
|
||||
"traduccion_id": t.ID,
|
||||
|
|
@ -235,8 +232,8 @@ func uploadToQdrant(translations []Translation, embeddings [][]float64) error {
|
|||
}
|
||||
|
||||
points = append(points, QdrantPoint{
|
||||
ID: pointID,
|
||||
Vector: embeddings[i],
|
||||
ID: pointIDs[i],
|
||||
Vector: t.Embedding,
|
||||
Payload: payload,
|
||||
})
|
||||
}
|
||||
|
|
@ -252,7 +249,12 @@ func uploadToQdrant(translations []Translation, embeddings [][]float64) error {
|
|||
}
|
||||
|
||||
url := fmt.Sprintf("%s/collections/%s/points", qdrantURL, collection)
|
||||
resp, err := http.Post(url, "application/json", bytes.NewReader(body))
|
||||
req, err := http.NewRequest(http.MethodPut, url, bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
@ -317,7 +319,7 @@ func main() {
|
|||
|
||||
ctx := context.Background()
|
||||
|
||||
if err := ensureCollection(); err != nil {
|
||||
if err := ensureCollection(ctx); err != nil {
|
||||
logger.Printf("Warning: Could not ensure collection: %v", err)
|
||||
}
|
||||
|
||||
|
|
@ -330,8 +332,8 @@ func main() {
|
|||
os.Exit(0)
|
||||
}()
|
||||
|
||||
logger.Printf("Config: qdrant=%s, ollama=%s, collection=%s, sleep=%ds, batch=%d",
|
||||
qdrantURL, ollamaURL, collection, sleepSec, batchSize)
|
||||
logger.Printf("Config: qdrant=%s, collection=%s, sleep=%ds, batch=%d",
|
||||
qdrantURL, collection, sleepSec, batchSize)
|
||||
|
||||
totalProcessed := 0
|
||||
|
||||
|
|
@ -351,30 +353,19 @@ func main() {
|
|||
|
||||
logger.Printf("Processing %d translations...", len(translations))
|
||||
|
||||
// Generate embeddings
|
||||
embeddings := make([][]float64, len(translations))
|
||||
for i, t := range translations {
|
||||
text := fmt.Sprintf("%s %s", t.Titulo, t.Resumen)
|
||||
emb, err := generateEmbedding(text)
|
||||
if err != nil {
|
||||
logger.Printf("Error generating embedding for %d: %v", t.ID, err)
|
||||
continue
|
||||
}
|
||||
embeddings[i] = emb
|
||||
}
|
||||
|
||||
// Upload to Qdrant
|
||||
if err := uploadToQdrant(translations, embeddings); err != nil {
|
||||
logger.Printf("Error uploading to Qdrant: %v", err)
|
||||
continue
|
||||
}
|
||||
|
||||
// Update DB status
|
||||
// Generate point IDs once (used both in Qdrant and DB)
|
||||
pointIDs := make([]string, len(translations))
|
||||
for i := range translations {
|
||||
pointIDs[i] = uuid.New().String()
|
||||
}
|
||||
|
||||
// Upload to Qdrant
|
||||
if err := uploadToQdrant(translations, pointIDs); err != nil {
|
||||
logger.Printf("Error uploading to Qdrant: %v", err)
|
||||
continue
|
||||
}
|
||||
|
||||
// Update DB status
|
||||
if err := updateTranslationStatus(ctx, translations, pointIDs); err != nil {
|
||||
logger.Printf("Error updating status: %v", err)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -187,6 +187,7 @@ func main() {
|
|||
admin.POST("/entities/retype", handlers.PatchEntityTipo)
|
||||
admin.GET("/backup", handlers.BackupDatabase)
|
||||
admin.GET("/backup/news", handlers.BackupNewsZipped)
|
||||
admin.POST("/restore", handlers.RestoreDatabase)
|
||||
admin.GET("/users", handlers.GetUsers)
|
||||
admin.POST("/users/:id/promote", handlers.PromoteUser)
|
||||
admin.POST("/users/:id/demote", handlers.DemoteUser)
|
||||
|
|
|
|||
|
|
@ -1,6 +1,8 @@
|
|||
module github.com/rss2/backend
|
||||
|
||||
go 1.22
|
||||
go 1.23
|
||||
|
||||
toolchain go1.23.12
|
||||
|
||||
require (
|
||||
github.com/PuerkitoBio/goquery v1.9.2
|
||||
|
|
|
|||
158
backend/go.sum
Normal file
158
backend/go.sum
Normal file
|
|
@ -0,0 +1,158 @@
|
|||
github.com/PuerkitoBio/goquery v1.9.2 h1:4/wZksC3KgkQw7SQgkKotmKljk0M6V8TUvA8Wb4yPeE=
|
||||
github.com/PuerkitoBio/goquery v1.9.2/go.mod h1:GHPCaP0ODyyxqcNoFGYlAprUFH81NuRPd0GX3Zu2Mvk=
|
||||
github.com/andybalholm/cascadia v1.3.2 h1:3Xi6Dw5lHF15JtdcmAHD3i1+T8plmv7BQ/nsViSLyss=
|
||||
github.com/andybalholm/cascadia v1.3.2/go.mod h1:7gtRlve5FxPPgIgX36uWBX58OdBsSS6lUvCFb+h7KvU=
|
||||
github.com/bsm/ginkgo/v2 v2.7.0 h1:ItPMPH90RbmZJt5GtkcNvIRuGEdwlBItdNVoyzaNQao=
|
||||
github.com/bsm/ginkgo/v2 v2.7.0/go.mod h1:AiKlXPm7ItEHNc/2+OkrNG4E0ITzojb9/xWzvQ9XZ9w=
|
||||
github.com/bsm/gomega v1.26.0 h1:LhQm+AFcgV2M0WyKroMASzAzCAJVpAxQXv4SaI9a69Y=
|
||||
github.com/bsm/gomega v1.26.0/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0=
|
||||
github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1O2AihPM=
|
||||
github.com/bytedance/sonic v1.9.1 h1:6iJ6NqdoxCDr6mbY8h18oSO+cShGSMRGCEo7F2h0x8s=
|
||||
github.com/bytedance/sonic v1.9.1/go.mod h1:i736AoUSYt75HyZLoJW9ERYxcy6eaN6h4BZXU064P/U=
|
||||
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
|
||||
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
github.com/chenzhuoyu/base64x v0.0.0-20211019084208-fb5309c8db06/go.mod h1:DH46F32mSOjUmXrMHnKwZdA8wcEefY7UVqBKYGjpdQY=
|
||||
github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 h1:qSGYFH7+jGhDF8vLC+iwCD4WpbV1EBDSzWkJODFLams=
|
||||
github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311/go.mod h1:b583jCggY9gE99b6G5LEC39OIiVsWj+R97kbl5odCEk=
|
||||
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
|
||||
github.com/gabriel-vasile/mimetype v1.4.2 h1:w5qFW6JKBz9Y393Y4q372O9A7cUSequkh1Q7OhCmWKU=
|
||||
github.com/gabriel-vasile/mimetype v1.4.2/go.mod h1:zApsH/mKG4w07erKIaJPFiX0Tsq9BFQgN3qGY5GnNgA=
|
||||
github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE=
|
||||
github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI=
|
||||
github.com/gin-gonic/gin v1.9.1 h1:4idEAncQnU5cB7BeOkPtxjfCSye0AAm1R0RVIqJ+Jmg=
|
||||
github.com/gin-gonic/gin v1.9.1/go.mod h1:hPrL7YrpYKXt5YId3A/Tnip5kqbEAP+KLuI3SUcPTeU=
|
||||
github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s=
|
||||
github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
|
||||
github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA=
|
||||
github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY=
|
||||
github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY=
|
||||
github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY=
|
||||
github.com/go-playground/validator/v10 v10.14.0 h1:vgvQWe3XCz3gIeFDm/HnTIbj6UGmg/+t63MyGU2n5js=
|
||||
github.com/go-playground/validator/v10 v10.14.0/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU=
|
||||
github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU=
|
||||
github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
|
||||
github.com/golang-jwt/jwt/v5 v5.0.0 h1:1n1XNM9hk7O9mnQoNBGolZvzebBQ7p93ULHRc28XJUE=
|
||||
github.com/golang-jwt/jwt/v5 v5.0.0/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk=
|
||||
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
|
||||
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
|
||||
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
|
||||
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/gorilla/websocket v1.5.1 h1:gmztn0JnHVt9JZquRuzLw3g4wouNVzKL15iLr/zn/QY=
|
||||
github.com/gorilla/websocket v1.5.1/go.mod h1:x3kM2JMyaluk02fnUJpQuwD2dCS5NDG2ZHL0uE0tcaY=
|
||||
github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
|
||||
github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
|
||||
github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk=
|
||||
github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=
|
||||
github.com/jackc/pgx/v5 v5.4.3 h1:cxFyXhxlvAifxnkKKdlxv8XqUf59tDlYjnV5YYfsJJY=
|
||||
github.com/jackc/pgx/v5 v5.4.3/go.mod h1:Ig06C2Vu0t5qXC60W8sqIthScaEnFvojjj9dSljmHRA=
|
||||
github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk=
|
||||
github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
|
||||
github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
|
||||
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
|
||||
github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
|
||||
github.com/klauspost/cpuid/v2 v2.2.4 h1:acbojRNwl3o09bUq+yDCtZFc1aiwaAAxtcn8YkZXnvk=
|
||||
github.com/klauspost/cpuid/v2 v2.2.4/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY=
|
||||
github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0=
|
||||
github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk=
|
||||
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
|
||||
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
|
||||
github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q=
|
||||
github.com/leodido/go-urn v1.2.4/go.mod h1:7ZrI8mTSeBSHl/UaRyKQW1qZeMgak41ANeCNaVckg+4=
|
||||
github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA=
|
||||
github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
|
||||
github.com/mmcdole/gofeed v1.2.1 h1:tPbFN+mfOLcM1kDF1x2c/N68ChbdBatkppdzf/vDe1s=
|
||||
github.com/mmcdole/gofeed v1.2.1/go.mod h1:2wVInNpgmC85q16QTTuwbuKxtKkHLCDDtf0dCmnrNr4=
|
||||
github.com/mmcdole/goxpp v1.1.0 h1:WwslZNF7KNAXTFuzRtn/OKZxFLJAAyOA9w82mDz2ZGI=
|
||||
github.com/mmcdole/goxpp v1.1.0/go.mod h1:v+25+lT2ViuQ7mVxcncQ8ch1URund48oH+jhjiwEgS8=
|
||||
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
|
||||
github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
|
||||
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
|
||||
github.com/pelletier/go-toml/v2 v2.0.8 h1:0ctb6s9mE31h0/lhu+J6OPmVeDxJn+kYnJc2jZR9tGQ=
|
||||
github.com/pelletier/go-toml/v2 v2.0.8/go.mod h1:vuYfssBdrU2XDZ9bYydBu6t+6a6PYNcZljzZR9VXg+4=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/redis/go-redis/v9 v9.0.5 h1:CuQcn5HIEeK7BgElubPP8CGtE0KakrnbBSTLjathl5o=
|
||||
github.com/redis/go-redis/v9 v9.0.5/go.mod h1:WqMKv5vnQbRuZstUwxQI195wHy+t4PuXDOjzMvcuQHk=
|
||||
github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
|
||||
github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
|
||||
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
|
||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
|
||||
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
|
||||
github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
|
||||
github.com/stretchr/testify v1.8.3 h1:RP3t2pwF7cMEbC1dqtB6poj3niw/9gnV4Cjg5oW5gtY=
|
||||
github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
|
||||
github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI=
|
||||
github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08=
|
||||
github.com/ugorji/go/codec v1.2.11 h1:BMaWp1Bb6fHwEtbplGBGJ498wD+LKlNSl25MjdZY4dU=
|
||||
github.com/ugorji/go/codec v1.2.11/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg=
|
||||
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
|
||||
golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8=
|
||||
golang.org/x/arch v0.3.0 h1:02VY4/ZcO/gBOH6PUaoiptASxtXU10jazRCP865E97k=
|
||||
golang.org/x/arch v0.3.0/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
|
||||
golang.org/x/crypto v0.26.0 h1:RrRspgV4mU+YwB4FYnuBoKsUapNIL5cohGAmSH3azsw=
|
||||
golang.org/x/crypto v0.26.0/go.mod h1:GY7jblb9wI+FOo5y8/S2oY4zWP07AkOJ4+jxCqdqn54=
|
||||
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
|
||||
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
|
||||
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
|
||||
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
|
||||
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
|
||||
golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns=
|
||||
golang.org/x/net v0.28.0 h1:a9JDOJc5GMUJ0+UDqmLT86WiEy7iWyIhz8gz8E4e5hE=
|
||||
golang.org/x/net v0.28.0/go.mod h1:yqtgsTWOOnlGLG9GFRrK3++bGOUEkNBoHZc8MEDWPNg=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ=
|
||||
golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo=
|
||||
golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
|
||||
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
|
||||
golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
|
||||
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
|
||||
golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
|
||||
golang.org/x/text v0.17.0 h1:XtiM5bkSOt+ewxlOE/aE/AKEHibwj/6gvWMl9Rsh0Qc=
|
||||
golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
|
||||
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
|
||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg=
|
||||
google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4=
|
||||
|
|
@ -6,14 +6,15 @@ import (
|
|||
"context"
|
||||
"encoding/csv"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"os/exec"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/rss2/backend/internal/config"
|
||||
"github.com/rss2/backend/internal/db"
|
||||
|
|
@ -637,17 +638,73 @@ func PatchEntityTipo(c *gin.Context) {
|
|||
})
|
||||
}
|
||||
|
||||
// pgConnInfo holds PostgreSQL connection parameters for pg_dump
|
||||
type pgConnInfo struct {
|
||||
host, port, name, user, pass string
|
||||
}
|
||||
|
||||
// resolvePgConnInfo prefers DATABASE_URL (the variable the API container gets)
|
||||
// and falls back to the individual DB_* variables used by the workers.
|
||||
func resolvePgConnInfo() pgConnInfo {
|
||||
if dbURL := os.Getenv("DATABASE_URL"); dbURL != "" {
|
||||
if u, err := url.Parse(dbURL); err == nil && u.Hostname() != "" {
|
||||
info := pgConnInfo{
|
||||
host: u.Hostname(),
|
||||
user: u.User.Username(),
|
||||
name: strings.TrimPrefix(u.Path, "/"),
|
||||
port: u.Port(),
|
||||
}
|
||||
if p, ok := u.User.Password(); ok {
|
||||
info.pass = p
|
||||
}
|
||||
if info.port == "" {
|
||||
info.port = "5432"
|
||||
}
|
||||
if info.user == "" {
|
||||
info.user = "postgres"
|
||||
}
|
||||
if info.name == "" {
|
||||
info.name = "postgres"
|
||||
}
|
||||
return info
|
||||
}
|
||||
}
|
||||
|
||||
info := pgConnInfo{
|
||||
host: os.Getenv("DB_HOST"),
|
||||
port: os.Getenv("DB_PORT"),
|
||||
name: os.Getenv("DB_NAME"),
|
||||
user: os.Getenv("DB_USER"),
|
||||
pass: os.Getenv("DB_PASS"),
|
||||
}
|
||||
if info.host == "" {
|
||||
info.host = "db"
|
||||
}
|
||||
if info.port == "" {
|
||||
info.port = "5432"
|
||||
}
|
||||
if info.name == "" {
|
||||
info.name = "rss"
|
||||
}
|
||||
if info.user == "" {
|
||||
info.user = "rss"
|
||||
}
|
||||
return info
|
||||
}
|
||||
|
||||
// BackupDatabase runs pg_dump and returns the SQL as a downloadable file
|
||||
func BackupDatabase(c *gin.Context) {
|
||||
// Ejecutar pg_dump desde dentro del contenedor db para evitar problemas de red
|
||||
// Usamos docker exec para ejecutar el comando dentro del servicio db
|
||||
cmd := exec.Command("docker", "exec", "rss2_db", "pg_dump",
|
||||
"-U", os.Getenv("POSTGRES_USER"),
|
||||
"-d", os.Getenv("POSTGRES_DB"),
|
||||
info := resolvePgConnInfo()
|
||||
|
||||
cmd := exec.Command("pg_dump",
|
||||
"-h", info.host,
|
||||
"-p", info.port,
|
||||
"-U", info.user,
|
||||
"-d", info.name,
|
||||
"--no-password",
|
||||
"--format=plain",
|
||||
"--pghost=5432",
|
||||
)
|
||||
cmd.Env = append(os.Environ(), fmt.Sprintf("PGPASSWORD=%s", info.pass))
|
||||
|
||||
var out bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
|
|
@ -671,32 +728,16 @@ func BackupDatabase(c *gin.Context) {
|
|||
|
||||
// BackupNewsZipped performs a pg_dump of news tables and returns a ZIP file
|
||||
func BackupNewsZipped(c *gin.Context) {
|
||||
dbHost := os.Getenv("DB_HOST")
|
||||
if dbHost == "" {
|
||||
dbHost = "db"
|
||||
}
|
||||
dbPort := os.Getenv("DB_PORT")
|
||||
if dbPort == "" {
|
||||
dbPort = "5432"
|
||||
}
|
||||
dbName := os.Getenv("DB_NAME")
|
||||
if dbName == "" {
|
||||
dbName = "rss"
|
||||
}
|
||||
dbUser := os.Getenv("DB_USER")
|
||||
if dbUser == "" {
|
||||
dbUser = "rss"
|
||||
}
|
||||
dbPass := os.Getenv("DB_PASS")
|
||||
info := resolvePgConnInfo()
|
||||
|
||||
// Tables to backup
|
||||
tables := []string{"noticias", "traducciones", "tags", "tags_noticia"}
|
||||
|
||||
args := []string{
|
||||
"-h", dbHost,
|
||||
"-p", dbPort,
|
||||
"-U", dbUser,
|
||||
"-d", dbName,
|
||||
"-h", info.host,
|
||||
"-p", info.port,
|
||||
"-U", info.user,
|
||||
"-d", info.name,
|
||||
"--no-password",
|
||||
}
|
||||
|
||||
|
|
@ -705,7 +746,7 @@ func BackupNewsZipped(c *gin.Context) {
|
|||
}
|
||||
|
||||
cmd := exec.Command("pg_dump", args...)
|
||||
cmd.Env = append(os.Environ(), fmt.Sprintf("PGPASSWORD=%s", dbPass))
|
||||
cmd.Env = append(os.Environ(), fmt.Sprintf("PGPASSWORD=%s", info.pass))
|
||||
|
||||
var sqlOut bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
|
|
@ -748,3 +789,107 @@ func BackupNewsZipped(c *gin.Context) {
|
|||
c.Header("Cache-Control", "no-cache")
|
||||
c.Data(http.StatusOK, "application/zip", buf.Bytes())
|
||||
}
|
||||
|
||||
// RestoreDatabase accepts an uploaded .sql or .zip backup and restores it via psql
|
||||
func RestoreDatabase(c *gin.Context) {
|
||||
file, err := c.FormFile("file")
|
||||
if err != nil {
|
||||
c.JSON(http.StatusBadRequest, gin.H{
|
||||
"error": "No file uploaded",
|
||||
"message": "Se requiere un archivo .sql o .zip",
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
uploaded, err := file.Open()
|
||||
if err != nil {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": "Failed to open uploaded file"})
|
||||
return
|
||||
}
|
||||
defer uploaded.Close()
|
||||
|
||||
data, err := io.ReadAll(uploaded)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": "Failed to read uploaded file"})
|
||||
return
|
||||
}
|
||||
|
||||
var sqlData []byte
|
||||
lowerName := strings.ToLower(file.Filename)
|
||||
switch {
|
||||
case strings.HasSuffix(lowerName, ".sql"):
|
||||
sqlData = data
|
||||
case strings.HasSuffix(lowerName, ".zip"):
|
||||
zr, err := zip.NewReader(bytes.NewReader(data), int64(len(data)))
|
||||
if err != nil {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": "Invalid ZIP file", "message": err.Error()})
|
||||
return
|
||||
}
|
||||
found := false
|
||||
for _, zf := range zr.File {
|
||||
if zf.FileInfo().IsDir() || !strings.HasSuffix(strings.ToLower(zf.Name), ".sql") {
|
||||
continue
|
||||
}
|
||||
rc, err := zf.Open()
|
||||
if err != nil {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": "Failed to open SQL inside ZIP", "message": err.Error()})
|
||||
return
|
||||
}
|
||||
sqlData, err = io.ReadAll(rc)
|
||||
rc.Close()
|
||||
if err != nil {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": "Failed to read SQL inside ZIP", "message": err.Error()})
|
||||
return
|
||||
}
|
||||
found = true
|
||||
break
|
||||
}
|
||||
if !found {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": "No SQL file found inside ZIP"})
|
||||
return
|
||||
}
|
||||
default:
|
||||
c.JSON(http.StatusBadRequest, gin.H{
|
||||
"error": "Unsupported file type",
|
||||
"message": "Solo se permiten archivos .sql o .zip",
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
if len(sqlData) == 0 {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": "Empty backup file"})
|
||||
return
|
||||
}
|
||||
|
||||
info := resolvePgConnInfo()
|
||||
|
||||
cmd := exec.Command("psql",
|
||||
"-h", info.host,
|
||||
"-p", info.port,
|
||||
"-U", info.user,
|
||||
"-d", info.name,
|
||||
"--no-password",
|
||||
"-v", "ON_ERROR_STOP=1",
|
||||
)
|
||||
cmd.Env = append(os.Environ(), fmt.Sprintf("PGPASSWORD=%s", info.pass))
|
||||
cmd.Stdin = bytes.NewReader(sqlData)
|
||||
|
||||
var out bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
cmd.Stdout = &out
|
||||
cmd.Stderr = &stderr
|
||||
|
||||
if err := cmd.Run(); err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{
|
||||
"error": "Restore failed",
|
||||
"details": stderr.String(),
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
c.JSON(http.StatusOK, gin.H{
|
||||
"message": "Base de datos restaurada correctamente",
|
||||
"filename": file.Filename,
|
||||
"output": out.String(),
|
||||
})
|
||||
}
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ import (
|
|||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
|
|
@ -397,11 +398,46 @@ func ImportFeeds(c *gin.Context) {
|
|||
}
|
||||
|
||||
reader := csv.NewReader(strings.NewReader(string(content)))
|
||||
_, err = reader.Read()
|
||||
records, err := reader.ReadAll()
|
||||
if err != nil {
|
||||
c.JSON(http.StatusBadRequest, models.ErrorResponse{Error: "Invalid CSV format"})
|
||||
return
|
||||
}
|
||||
if len(records) == 0 {
|
||||
c.JSON(http.StatusBadRequest, models.ErrorResponse{Error: "Empty CSV file"})
|
||||
return
|
||||
}
|
||||
|
||||
colIndex := func(header []string, names ...string) int {
|
||||
for i, h := range header {
|
||||
h = strings.ToLower(strings.TrimSpace(h))
|
||||
for _, n := range names {
|
||||
if h == n {
|
||||
return i
|
||||
}
|
||||
}
|
||||
}
|
||||
return -1
|
||||
}
|
||||
|
||||
header := records[0]
|
||||
isHeader := colIndex(header, "url", "link", "feed", "nombre", "name", "titulo") != -1
|
||||
|
||||
var dataRows [][]string
|
||||
iNombre, iDesc, iURL, iCat, iPais, iLang, iActivo, iFallos := -1, -1, -1, -1, -1, -1, -1, -1
|
||||
if isHeader {
|
||||
iNombre = colIndex(header, "nombre", "name", "titulo", "title")
|
||||
iDesc = colIndex(header, "descripcion", "description")
|
||||
iURL = colIndex(header, "url", "link", "feed")
|
||||
iCat = colIndex(header, "categoria_id", "category_id", "categoria", "category")
|
||||
iPais = colIndex(header, "pais_id", "country_id", "pais", "country")
|
||||
iLang = colIndex(header, "idioma", "language", "lang")
|
||||
iActivo = colIndex(header, "activo", "active", "enabled")
|
||||
iFallos = colIndex(header, "fallos", "failures", "fails")
|
||||
dataRows = records[1:]
|
||||
} else {
|
||||
dataRows = records
|
||||
}
|
||||
|
||||
imported := 0
|
||||
skipped := 0
|
||||
|
|
@ -415,71 +451,115 @@ func ImportFeeds(c *gin.Context) {
|
|||
}
|
||||
defer tx.Rollback(context.Background())
|
||||
|
||||
for {
|
||||
record, err := reader.Read()
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
failed++
|
||||
continue
|
||||
field := func(record []string, idx int) string {
|
||||
if idx < 0 || idx >= len(record) {
|
||||
return ""
|
||||
}
|
||||
return strings.TrimSpace(record[idx])
|
||||
}
|
||||
|
||||
if len(record) < 4 {
|
||||
for _, record := range dataRows {
|
||||
if len(record) == 0 {
|
||||
skipped++
|
||||
continue
|
||||
}
|
||||
|
||||
nombre := strings.TrimSpace(record[1])
|
||||
url := strings.TrimSpace(record[3])
|
||||
var nombre, url string
|
||||
if isHeader {
|
||||
nombre = field(record, iNombre)
|
||||
url = field(record, iURL)
|
||||
} else if len(record) == 1 {
|
||||
url = field(record, 0)
|
||||
} else if len(record) == 2 {
|
||||
a := field(record, 0)
|
||||
b := field(record, 1)
|
||||
if looksLikeURL(a) {
|
||||
url, nombre = a, b
|
||||
} else {
|
||||
nombre, url = a, b
|
||||
}
|
||||
} else {
|
||||
nombre = field(record, 1)
|
||||
url = field(record, 3)
|
||||
}
|
||||
|
||||
if nombre == "" || url == "" {
|
||||
if url == "" {
|
||||
skipped++
|
||||
continue
|
||||
}
|
||||
if nombre == "" {
|
||||
nombre = nombreFromURL(url)
|
||||
}
|
||||
|
||||
var descripcion *string
|
||||
if len(record) > 2 && strings.TrimSpace(record[2]) != "" {
|
||||
descripcionStr := strings.TrimSpace(record[2])
|
||||
descripcion = &descripcionStr
|
||||
}
|
||||
|
||||
var categoriaID *int64
|
||||
if len(record) > 4 && strings.TrimSpace(record[4]) != "" {
|
||||
catID, err := strconv.ParseInt(strings.TrimSpace(record[4]), 10, 64)
|
||||
if err == nil {
|
||||
categoriaID = &catID
|
||||
}
|
||||
}
|
||||
|
||||
var paisID *int64
|
||||
if len(record) > 6 && strings.TrimSpace(record[6]) != "" {
|
||||
pID, err := strconv.ParseInt(strings.TrimSpace(record[6]), 10, 64)
|
||||
if err == nil {
|
||||
paisID = &pID
|
||||
}
|
||||
}
|
||||
|
||||
var idioma *string
|
||||
if len(record) > 8 && strings.TrimSpace(record[8]) != "" {
|
||||
lang := strings.TrimSpace(record[8])
|
||||
if len(lang) > 2 {
|
||||
lang = lang[:2]
|
||||
}
|
||||
idioma = &lang
|
||||
}
|
||||
|
||||
activo := true
|
||||
if len(record) > 9 && strings.TrimSpace(record[9]) != "" {
|
||||
activo = strings.ToLower(strings.TrimSpace(record[9])) == "true"
|
||||
var fallos int64
|
||||
|
||||
if isHeader {
|
||||
if d := field(record, iDesc); d != "" {
|
||||
descripcion = &d
|
||||
}
|
||||
if v := field(record, iCat); v != "" {
|
||||
if id, err := strconv.ParseInt(v, 10, 64); err == nil {
|
||||
categoriaID = &id
|
||||
}
|
||||
}
|
||||
if v := field(record, iPais); v != "" {
|
||||
if id, err := strconv.ParseInt(v, 10, 64); err == nil {
|
||||
paisID = &id
|
||||
}
|
||||
}
|
||||
if l := field(record, iLang); l != "" {
|
||||
if len(l) > 2 {
|
||||
l = l[:2]
|
||||
}
|
||||
idioma = &l
|
||||
}
|
||||
if a := field(record, iActivo); a != "" {
|
||||
activo = strings.ToLower(a) == "true"
|
||||
}
|
||||
if f := field(record, iFallos); f != "" {
|
||||
if v, err := strconv.ParseInt(f, 10, 64); err == nil {
|
||||
fallos = v
|
||||
}
|
||||
}
|
||||
} else {
|
||||
if d := field(record, 2); d != "" {
|
||||
descripcion = &d
|
||||
}
|
||||
if v := field(record, 4); v != "" {
|
||||
if id, err := strconv.ParseInt(v, 10, 64); err == nil {
|
||||
categoriaID = &id
|
||||
}
|
||||
}
|
||||
if v := field(record, 6); v != "" {
|
||||
if id, err := strconv.ParseInt(v, 10, 64); err == nil {
|
||||
paisID = &id
|
||||
}
|
||||
}
|
||||
if l := field(record, 8); l != "" {
|
||||
if len(l) > 2 {
|
||||
l = l[:2]
|
||||
}
|
||||
idioma = &l
|
||||
}
|
||||
if a := field(record, 9); a != "" {
|
||||
activo = strings.ToLower(a) == "true"
|
||||
}
|
||||
if f := field(record, 10); f != "" {
|
||||
if v, err := strconv.ParseInt(f, 10, 64); err == nil {
|
||||
fallos = v
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var fallos int64
|
||||
if len(record) > 10 && strings.TrimSpace(record[10]) != "" {
|
||||
f, err := strconv.ParseInt(strings.TrimSpace(record[10]), 10, 64)
|
||||
if err == nil {
|
||||
fallos = f
|
||||
}
|
||||
if _, err := tx.Exec(context.Background(), "SAVEPOINT sp"); err != nil {
|
||||
failed++
|
||||
errors = append(errors, fmt.Sprintf("Error for %s: %v", url, err))
|
||||
continue
|
||||
}
|
||||
|
||||
var existingID int64
|
||||
|
|
@ -490,22 +570,25 @@ func ImportFeeds(c *gin.Context) {
|
|||
WHERE id=$8`,
|
||||
nombre, descripcion, categoriaID, paisID, idioma, activo, fallos, existingID,
|
||||
)
|
||||
if err != nil {
|
||||
failed++
|
||||
errors = append(errors, fmt.Sprintf("Error updating %s: %v", url, err))
|
||||
continue
|
||||
}
|
||||
} else {
|
||||
_, err = tx.Exec(context.Background(), `
|
||||
INSERT INTO feeds (nombre, descripcion, url, categoria_id, pais_id, idioma, activo, fallos)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8)`,
|
||||
nombre, descripcion, url, categoriaID, paisID, idioma, activo, fallos,
|
||||
)
|
||||
if err != nil {
|
||||
failed++
|
||||
errors = append(errors, fmt.Sprintf("Error inserting %s: %v", url, err))
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
tx.Exec(context.Background(), "ROLLBACK TO SAVEPOINT sp")
|
||||
failed++
|
||||
errors = append(errors, fmt.Sprintf("Error upserting %s: %v", url, err))
|
||||
continue
|
||||
}
|
||||
|
||||
if _, err := tx.Exec(context.Background(), "RELEASE SAVEPOINT sp"); err != nil {
|
||||
failed++
|
||||
errors = append(errors, fmt.Sprintf("Error for %s: %v", url, err))
|
||||
continue
|
||||
}
|
||||
|
||||
imported++
|
||||
|
|
@ -525,6 +608,22 @@ func ImportFeeds(c *gin.Context) {
|
|||
})
|
||||
}
|
||||
|
||||
func looksLikeURL(s string) bool {
|
||||
return strings.HasPrefix(s, "http://") || strings.HasPrefix(s, "https://") || strings.HasPrefix(s, "http://www.") || strings.HasPrefix(s, "https://www.") || strings.HasPrefix(s, "www.") || strings.HasPrefix(s, "feed://")
|
||||
}
|
||||
|
||||
func nombreFromURL(u string) string {
|
||||
u = strings.TrimSpace(u)
|
||||
if !strings.HasPrefix(u, "http://") && !strings.HasPrefix(u, "https://") {
|
||||
u = "https://" + u
|
||||
}
|
||||
parsed, err := url.Parse(u)
|
||||
if err != nil || parsed.Host == "" {
|
||||
return u
|
||||
}
|
||||
return strings.TrimPrefix(parsed.Host, "www.")
|
||||
}
|
||||
|
||||
func stringOrEmpty(s *string) string {
|
||||
if s == nil {
|
||||
return ""
|
||||
|
|
|
|||
|
|
@ -51,55 +51,45 @@ func SearchNews(c *gin.Context) {
|
|||
offset := (page - 1) * perPage
|
||||
|
||||
// Build dynamic query
|
||||
args := []interface{}{}
|
||||
argNum := 1
|
||||
args := []interface{}{lang}
|
||||
argNum := 2
|
||||
whereClause := "WHERE 1=1"
|
||||
|
||||
if query != "" {
|
||||
whereClause += " AND (n.titulo ILIKE $" + strconv.Itoa(argNum) + " OR n.resumen ILIKE $" + strconv.Itoa(argNum) + " OR n.contenido ILIKE $" + strconv.Itoa(argNum) + ")"
|
||||
whereClause += " AND (n.titulo ILIKE $" + strconv.Itoa(argNum) + " OR n.resumen ILIKE $" + strconv.Itoa(argNum) + ")"
|
||||
args = append(args, "%"+query+"%")
|
||||
argNum++
|
||||
}
|
||||
|
||||
if lang != "" {
|
||||
whereClause += " AND t.lang_to = $" + strconv.Itoa(argNum)
|
||||
args = append(args, lang)
|
||||
argNum++
|
||||
}
|
||||
|
||||
if categoriaID != "" {
|
||||
whereClause += " AND n.categoria_id = $" + strconv.Itoa(argNum)
|
||||
catID, err := strconv.ParseInt(categoriaID, 10, 64)
|
||||
if err == nil {
|
||||
if catID, err := strconv.ParseInt(categoriaID, 10, 64); err == nil {
|
||||
whereClause += " AND n.categoria_id = $" + strconv.Itoa(argNum)
|
||||
args = append(args, catID)
|
||||
argNum++
|
||||
}
|
||||
}
|
||||
|
||||
if paisID != "" {
|
||||
whereClause += " AND n.pais_id = $" + strconv.Itoa(argNum)
|
||||
pID, err := strconv.ParseInt(paisID, 10, 64)
|
||||
if err == nil {
|
||||
if pID, err := strconv.ParseInt(paisID, 10, 64); err == nil {
|
||||
whereClause += " AND n.pais_id = $" + strconv.Itoa(argNum)
|
||||
args = append(args, pID)
|
||||
argNum++
|
||||
}
|
||||
}
|
||||
|
||||
args = append(args, perPage, offset)
|
||||
|
||||
sqlQuery := `
|
||||
SELECT n.id, n.titulo, n.resumen, n.contenido, n.url, n.imagen,
|
||||
n.feed_id, n.lang, n.categoria_id, n.pais_id, n.created_at, n.updated_at,
|
||||
COALESCE(t.titulo_trad, '') as titulo_trad,
|
||||
COALESCE(t.resumen_trad, '') as resumen_trad,
|
||||
t.lang_to as lang_trad,
|
||||
f.nombre as fuente_nombre
|
||||
SELECT n.id, n.titulo, COALESCE(n.resumen, ''), n.url, n.fecha, n.imagen_url,
|
||||
n.categoria_id, n.pais_id, n.fuente_nombre,
|
||||
t.titulo_trad,
|
||||
t.resumen_trad,
|
||||
t.lang_to as lang_trad
|
||||
FROM noticias n
|
||||
LEFT JOIN traducciones t ON t.noticia_id = n.id AND t.lang_to = $` + strconv.Itoa(argNum) + `
|
||||
LEFT JOIN feeds f ON f.id = n.feed_id
|
||||
LEFT JOIN traducciones t ON t.noticia_id = n.id AND t.lang_to = $1
|
||||
` + whereClause + `
|
||||
ORDER BY n.created_at DESC
|
||||
LIMIT $` + strconv.Itoa(argNum+1) + ` OFFSET $` + strconv.Itoa(argNum+2)
|
||||
ORDER BY n.fecha DESC
|
||||
LIMIT $` + strconv.Itoa(argNum) + ` OFFSET $` + strconv.Itoa(argNum+1)
|
||||
|
||||
args = append(args, perPage, offset)
|
||||
|
||||
rows, err := db.GetPool().Query(ctx, sqlQuery, args...)
|
||||
if err != nil {
|
||||
|
|
@ -108,33 +98,38 @@ func SearchNews(c *gin.Context) {
|
|||
}
|
||||
defer rows.Close()
|
||||
|
||||
var newsList []models.NewsWithTranslations
|
||||
var newsList []NewsResponse
|
||||
for rows.Next() {
|
||||
var n models.NewsWithTranslations
|
||||
var imagen *string
|
||||
var n NewsResponse
|
||||
var imagenURL, fuenteNombre *string
|
||||
var categoriaIDp, paisIDp *int64
|
||||
|
||||
err := rows.Scan(
|
||||
&n.ID, &n.Title, &n.Summary, &n.Content, &n.URL, &imagen,
|
||||
&n.FeedID, &n.Lang, &n.CategoryID, &n.CountryID, &n.CreatedAt, &n.UpdatedAt,
|
||||
&n.ID, &n.Titulo, &n.Resumen, &n.URL, &n.Fecha, &imagenURL,
|
||||
&categoriaIDp, &paisIDp, &fuenteNombre,
|
||||
&n.TitleTranslated, &n.SummaryTranslated, &n.LangTranslated,
|
||||
)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if imagen != nil {
|
||||
n.ImageURL = imagen
|
||||
if imagenURL != nil {
|
||||
n.ImagenURL = imagenURL
|
||||
}
|
||||
if fuenteNombre != nil {
|
||||
n.FuenteNombre = *fuenteNombre
|
||||
}
|
||||
n.CategoriaID = categoriaIDp
|
||||
n.PaisID = paisIDp
|
||||
newsList = append(newsList, n)
|
||||
}
|
||||
|
||||
// Get total count
|
||||
countArgs := args[:len(args)-2]
|
||||
|
||||
// Remove LIMIT/OFFSET from args for count
|
||||
var total int
|
||||
err = db.GetPool().QueryRow(ctx, `
|
||||
SELECT COUNT(*) FROM noticias n
|
||||
LEFT JOIN traducciones t ON t.noticia_id = n.id AND t.lang_to = $`+strconv.Itoa(argNum)+`
|
||||
LEFT JOIN traducciones t ON t.noticia_id = n.id AND t.lang_to = $1
|
||||
`+whereClause, countArgs...).Scan(&total)
|
||||
if err != nil {
|
||||
total = len(newsList)
|
||||
|
|
@ -142,15 +137,13 @@ func SearchNews(c *gin.Context) {
|
|||
|
||||
totalPages := (total + perPage - 1) / perPage
|
||||
|
||||
response := models.NewsListResponse{
|
||||
News: newsList,
|
||||
Total: total,
|
||||
Page: page,
|
||||
PerPage: perPage,
|
||||
TotalPages: totalPages,
|
||||
}
|
||||
|
||||
c.JSON(http.StatusOK, response)
|
||||
c.JSON(http.StatusOK, gin.H{
|
||||
"news": newsList,
|
||||
"total": total,
|
||||
"page": page,
|
||||
"per_page": perPage,
|
||||
"total_pages": totalPages,
|
||||
})
|
||||
}
|
||||
|
||||
func GetStats(c *gin.Context) {
|
||||
|
|
|
|||
|
|
@ -132,7 +132,10 @@ func writeLoop(wsWorker *WSWorker) {
|
|||
for {
|
||||
select {
|
||||
case <-ticker.C:
|
||||
if err := wsWorker.conn.WriteMessage(websocket.PingMessage, nil); err != nil {
|
||||
wsWorker.mu.Lock()
|
||||
err := wsWorker.conn.WriteMessage(websocket.PingMessage, nil)
|
||||
wsWorker.mu.Unlock()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
|
|
@ -163,7 +166,9 @@ func heartbeatLoop(wsWorker *WSWorker) {
|
|||
|
||||
func sendWS(wsWorker *WSWorker, msg models.WSServerMessage) {
|
||||
data, _ := json.Marshal(msg)
|
||||
wsWorker.mu.Lock()
|
||||
wsWorker.conn.WriteMessage(websocket.TextMessage, data)
|
||||
wsWorker.mu.Unlock()
|
||||
}
|
||||
|
||||
func cleanupWorker(wsWorker *WSWorker, apiKey string) {
|
||||
|
|
|
|||
16
backup.sh
Executable file
16
backup.sh
Executable file
|
|
@ -0,0 +1,16 @@
|
|||
#!/bin/bash
|
||||
# Backup diario de PostgreSQL rss2
|
||||
set -e
|
||||
|
||||
BACKUP_DIR="/home/x/rss2/backups"
|
||||
TIMESTAMP=$(date +%Y%m%d_%H%M%S)
|
||||
OUT="$BACKUP_DIR/rss-${TIMESTAMP}.sql.gz"
|
||||
LOG="$BACKUP_DIR/backup.log"
|
||||
|
||||
mkdir -p "$BACKUP_DIR"
|
||||
docker exec rss2_db pg_dump -U rss -d rss --no-owner --no-privileges | gzip > "$OUT"
|
||||
|
||||
# Retener solo los ultimos 30 dias (borrado seguro de ficheros concretos, nunca rm -rf)
|
||||
find "$BACKUP_DIR" -maxdepth 1 -name 'rss-*.sql.gz' -mtime +30 -delete
|
||||
|
||||
echo "$(date '+%F %T') Backup OK: $OUT ($(du -h "$OUT" | cut -f1))" >> "$LOG"
|
||||
10
docker-compose.local.yml
Normal file
10
docker-compose.local.yml
Normal file
|
|
@ -0,0 +1,10 @@
|
|||
# Override local: permite acceder a Grafana en http://localhost:3001
|
||||
# La red monitoring es internal:true y compose descarta los puertos de servicios
|
||||
# que solo están en redes internas. Añadir la red frontend (no internal) permite
|
||||
# publicar el puerto 127.0.0.1:3001.
|
||||
# Uso: docker compose -f docker-compose.yml -f docker-compose.local.yml up -d
|
||||
services:
|
||||
grafana:
|
||||
networks:
|
||||
- monitoring
|
||||
- frontend
|
||||
|
|
@ -308,6 +308,7 @@ services:
|
|||
CT2_MODEL_PATH: /app/models/nllb-ct2
|
||||
CT2_DEVICE: cpu
|
||||
CT2_COMPUTE_TYPE: int8
|
||||
CT2_INTRA_THREADS: 4
|
||||
UNIVERSAL_MODEL: facebook/nllb-200-distilled-600M
|
||||
HF_HOME: /app/hf_cache
|
||||
TZ: Europe/Madrid
|
||||
|
|
@ -325,14 +326,13 @@ services:
|
|||
restart: unless-stopped
|
||||
|
||||
# ==================================================================================
|
||||
# TRANSLATOR GPU - Multiple instances with --scale for parallel GPU processing
|
||||
# Configure multiple GPU workers: docker compose up -d --scale translator-gpu=2
|
||||
# TRANSLATOR CPU Nº2 - Segundo worker CPU (paralelo al translator)
|
||||
# ==================================================================================
|
||||
translator-gpu:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile.translator-gpu
|
||||
image: rss2-translator-gpu:latest
|
||||
dockerfile: Dockerfile.translator
|
||||
image: rss2-translator:latest
|
||||
command: bash -lc "python -m workers.ctranslator_worker"
|
||||
environment:
|
||||
DB_HOST: db
|
||||
|
|
@ -341,70 +341,27 @@ services:
|
|||
DB_USER: ${DB_USER:-rss}
|
||||
DB_PASS: ${DB_PASS}
|
||||
TARGET_LANGS: es
|
||||
TRANSLATOR_BATCH: 256
|
||||
CT2_MODEL_PATH: /app/models/nllb-ct2-1.3b
|
||||
CT2_DEVICE: cuda
|
||||
CT2_COMPUTE_TYPE: float16
|
||||
UNIVERSAL_MODEL: facebook/nllb-200-1.3B
|
||||
TRANSLATOR_BATCH: 32
|
||||
CT2_MODEL_PATH: /app/models/nllb-ct2
|
||||
CT2_DEVICE: cpu
|
||||
CT2_COMPUTE_TYPE: int8
|
||||
CT2_INTRA_THREADS: 4
|
||||
UNIVERSAL_MODEL: facebook/nllb-200-distilled-600M
|
||||
HF_HOME: /app/hf_cache
|
||||
TZ: Europe/Madrid
|
||||
TRANSLATOR_ID: ${TRANSLATOR_ID:-}
|
||||
PYTORCH_CUDA_ALLOC_CONF: max_split_size_mb:512
|
||||
NCCL_DEBUG: INFO
|
||||
PYTORCH_ENABLE_MPS_FALLBACK: 1
|
||||
volumes:
|
||||
- ./workers:/app/workers
|
||||
- ./hf_cache:/app/hf_cache
|
||||
- ./models:/app/models
|
||||
- /dev/nvidia:/dev/nvidia
|
||||
- /usr/local/nvidia:/usr/local/nvidia:ro
|
||||
networks:
|
||||
- backend
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: 1
|
||||
capabilities: [gpu]
|
||||
depends_on:
|
||||
db:
|
||||
condition: service_healthy
|
||||
restart: unless-stopped
|
||||
|
||||
# ==================================================================================
|
||||
# REMOTE TRANSLATOR WORKER - Worker remoto que se conecta por WebSocket
|
||||
# ==================================================================================
|
||||
remote-translator:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile.remote-worker
|
||||
container_name: rss2_remote_translator
|
||||
environment:
|
||||
WORKER_NAME: ${WORKER_NAME:-remote-worker-1}
|
||||
WORKER_API_KEY: ${WORKER_API_KEY:-}
|
||||
WORKER_SERVER: ${WORKER_SERVER:-ws://backend-go:8080/ws/worker}
|
||||
CT2_DEVICE: ${CT2_DEVICE:-cuda}
|
||||
CT2_MODEL_PATH: /app/models/nllb-ct2
|
||||
CT2_COMPUTE_TYPE: ${CT2_COMPUTE_TYPE:-float16}
|
||||
UNIVERSAL_MODEL: facebook/nllb-200-1.3B
|
||||
HF_HOME: /app/hf_cache
|
||||
TZ: Europe/Madrid
|
||||
volumes:
|
||||
- ./hf_cache:/app/hf_cache
|
||||
- ./models:/app/models
|
||||
networks:
|
||||
- backend
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
memory: 6G
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: 1
|
||||
capabilities: [ gpu ]
|
||||
restart: unless-stopped
|
||||
|
||||
# ==================================================================================
|
||||
# TRANSLATION SCHEDULER - Creates translation jobs
|
||||
# ==================================================================================
|
||||
|
|
@ -455,7 +412,7 @@ services:
|
|||
EMB_SLEEP_IDLE: 5
|
||||
EMB_LANGS: es
|
||||
EMB_LIMIT: 1000
|
||||
DEVICE: cuda
|
||||
DEVICE: cpu
|
||||
HF_HOME: /app/hf_cache
|
||||
TZ: Europe/Madrid
|
||||
volumes:
|
||||
|
|
@ -467,11 +424,6 @@ services:
|
|||
resources:
|
||||
limits:
|
||||
memory: 6G
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: 1
|
||||
capabilities: [ gpu ]
|
||||
depends_on:
|
||||
db:
|
||||
condition: service_healthy
|
||||
|
|
@ -523,7 +475,7 @@ services:
|
|||
RELATED_SLEEP: 10
|
||||
RELATED_BATCH: 200
|
||||
RELATED_TOPK: 10
|
||||
EMB_MODEL: mxbai-embed-large
|
||||
EMB_MODEL: sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2
|
||||
TZ: Europe/Madrid
|
||||
networks:
|
||||
- backend
|
||||
|
|
@ -660,117 +612,8 @@ services:
|
|||
memory: 2G
|
||||
|
||||
# ==================================================================================
|
||||
# LLM CATEGORIZER (Python) - Categorización con Ollama
|
||||
# REDES SEGMENTADAS
|
||||
# ==================================================================================
|
||||
llm-categorizer:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile
|
||||
container_name: rss2_llm_categorizer
|
||||
command: bash -lc "python -m workers.simple_categorizer_worker"
|
||||
environment:
|
||||
DB_HOST: db
|
||||
DB_PORT: 5432
|
||||
DB_NAME: ${DB_NAME:-rss}
|
||||
DB_USER: ${DB_USER:-rss}
|
||||
DB_PASS: ${DB_PASS}
|
||||
CATEGORIZER_BATCH_SIZE: 10
|
||||
CATEGORIZER_SLEEP_IDLE: 5
|
||||
TZ: Europe/Madrid
|
||||
volumes:
|
||||
- ./workers:/app/workers
|
||||
networks:
|
||||
- backend
|
||||
depends_on:
|
||||
db:
|
||||
condition: service_healthy
|
||||
restart: unless-stopped
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: '2'
|
||||
memory: 1G
|
||||
|
||||
# ==================================================================================
|
||||
# MONITORING STACK - SECURED
|
||||
# ==================================================================================
|
||||
|
||||
prometheus:
|
||||
image: prom/prometheus:latest
|
||||
container_name: rss2_prometheus
|
||||
volumes:
|
||||
- ./monitoring/prometheus.yml:/etc/prometheus/prometheus.yml:ro
|
||||
- prometheus_data:/prometheus
|
||||
command:
|
||||
- '--config.file=/etc/prometheus/prometheus.yml'
|
||||
- '--storage.tsdb.path=/prometheus'
|
||||
- '--web.console.libraries=/usr/share/prometheus/console_libraries'
|
||||
- '--web.console.templates=/usr/share/prometheus/consoles'
|
||||
# SEGURIDAD: Sin exposición de puertos - acceso solo vía Grafana o túnel SSH
|
||||
# ports:
|
||||
# - "9090:9090"
|
||||
networks:
|
||||
- monitoring
|
||||
restart: unless-stopped
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: '1'
|
||||
memory: 2G
|
||||
|
||||
grafana:
|
||||
image: grafana/grafana:latest
|
||||
container_name: rss2_grafana
|
||||
# SEGURIDAD: Acceso solo en localhost o vía túnel SSH
|
||||
# Para acceso remoto, usar túnel SSH: ssh -L 3001:localhost:3001 user@server
|
||||
ports:
|
||||
- "127.0.0.1:3001:3000"
|
||||
environment:
|
||||
# SEGURIDAD: Cambiar este password en producción
|
||||
- GF_SECURITY_ADMIN_PASSWORD=${GRAFANA_PASSWORD:-change_this_password}
|
||||
- GF_USERS_ALLOW_SIGN_UP=false
|
||||
- GF_SERVER_ROOT_URL=http://localhost:3001
|
||||
- GF_SECURITY_COOKIE_SECURE=false
|
||||
- GF_SECURITY_COOKIE_SAMESITE=lax
|
||||
volumes:
|
||||
- grafana_data:/var/lib/grafana
|
||||
networks:
|
||||
- monitoring
|
||||
depends_on:
|
||||
- prometheus
|
||||
restart: unless-stopped
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: '1'
|
||||
memory: 1G
|
||||
|
||||
cadvisor:
|
||||
image: gcr.io/cadvisor/cadvisor:latest
|
||||
container_name: rss2_cadvisor
|
||||
# SEGURIDAD: Sin exposición de puertos - solo acceso interno
|
||||
# ports:
|
||||
# - "8081:8080"
|
||||
volumes:
|
||||
- /:/rootfs:ro
|
||||
- /var/run:/var/run:ro
|
||||
- /sys:/sys:ro
|
||||
- /var/lib/docker/:/var/lib/docker:ro
|
||||
- /dev/disk/:/dev/disk:ro
|
||||
devices:
|
||||
- /dev/kmsg
|
||||
networks:
|
||||
- monitoring
|
||||
restart: unless-stopped
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: '0.5'
|
||||
memory: 512M
|
||||
|
||||
# ==================================================================================
|
||||
# REDES SEGMENTADAS
|
||||
# ==================================================================================
|
||||
networks:
|
||||
# Red frontal - Solo nginx y web app
|
||||
frontend:
|
||||
|
|
@ -784,13 +627,5 @@ networks:
|
|||
driver: bridge
|
||||
internal: false # Acceso externo permitido (necesario para ingestor)
|
||||
|
||||
# Red de monitoreo - Prometheus, Grafana, cAdvisor
|
||||
monitoring:
|
||||
name: rss2_monitoring
|
||||
driver: bridge
|
||||
internal: true
|
||||
|
||||
volumes:
|
||||
prometheus_data:
|
||||
grafana_data:
|
||||
torch_extensions:
|
||||
|
|
|
|||
|
|
@ -1,27 +1,23 @@
|
|||
#!/bin/bash
|
||||
set -e
|
||||
|
||||
# Detectar si la base de datos necesita reinicialización
|
||||
# Directorio de datos de PostgreSQL
|
||||
PGDATA_DIR="/var/lib/postgresql/data/18/main"
|
||||
|
||||
echo "RSS2: Checking database integrity..."
|
||||
echo "RSS2: Checking database presence..."
|
||||
|
||||
# Si no existe el archivo de versión, es una base de datos nueva
|
||||
if [ ! -f "$PGDATA_DIR/PG_VERSION" ]; then
|
||||
echo "RSS2: New database - will be initialized by docker-entrypoint"
|
||||
# REGLA: nunca se borra ni se modifica el directorio de datos automáticamente.
|
||||
# Solo se informa del estado y se deja que el entrypoint oficial de PostgreSQL
|
||||
# inicialice (si no existe) o arranque con los datos existentes.
|
||||
if [ -f "$PGDATA_DIR/PG_VERSION" ]; then
|
||||
echo "RSS2: Existing database found at $PGDATA_DIR - starting normally"
|
||||
else
|
||||
# Verificar si la base de datos es funcional
|
||||
if ! pg_isready -h localhost -p 5432 -U "${POSTGRES_USER:-rss}" 2>/dev/null; then
|
||||
echo "RSS2: Database appears corrupted - removing old data files for fresh initialization..."
|
||||
# Eliminar solo los archivos de datos, no todo el directorio
|
||||
rm -rf "$PGDATA_DIR"/*
|
||||
echo "RSS2: Data files removed - docker-entrypoint will initialize fresh database"
|
||||
else
|
||||
echo "RSS2: Database is healthy"
|
||||
fi
|
||||
echo "RSS2: No database found at $PGDATA_DIR - docker-entrypoint will initialize it"
|
||||
fi
|
||||
|
||||
# Ejecutar el entrypoint original con los parámetros de PostgreSQL
|
||||
# Ejecutar el entrypoint original con los parámetros de PostgreSQL.
|
||||
# Si la base de datos estuviera corrupta, PostgreSQL fallará al arrancar y
|
||||
# registrará el error en los logs; NO se elimina ningún dato de forma automática.
|
||||
exec docker-entrypoint.sh \
|
||||
postgres \
|
||||
-c max_connections=200 \
|
||||
|
|
|
|||
|
|
@ -1,9 +1,11 @@
|
|||
import { useState } from 'react'
|
||||
import { apiService } from '../services/api'
|
||||
import { AlertTriangle, Database, RefreshCw, Download, FileArchive } from 'lucide-react'
|
||||
import { AlertTriangle, Database, RefreshCw, Download, FileArchive, Upload } from 'lucide-react'
|
||||
|
||||
export function AdminSettings() {
|
||||
const [resetting, setResetting] = useState(false)
|
||||
const [restoring, setRestoring] = useState(false)
|
||||
const [restoreFile, setRestoreFile] = useState<File | null>(null)
|
||||
const [message, setMessage] = useState<{ type: 'success' | 'error'; text: string } | null>(null)
|
||||
|
||||
const handleReset = async () => {
|
||||
|
|
@ -35,6 +37,31 @@ export function AdminSettings() {
|
|||
}
|
||||
}
|
||||
|
||||
const handleRestore = async () => {
|
||||
if (!restoreFile) {
|
||||
setMessage({ type: 'error', text: 'Selecciona un archivo de copia de seguridad (.sql o .zip)' })
|
||||
return
|
||||
}
|
||||
|
||||
if (!confirm('¿Restaurar la base de datos?\n\nEsta acción sobrescribirá los datos existentes con el contenido del archivo de copia de seguridad. No se puede deshacer.')) return
|
||||
|
||||
setRestoring(true)
|
||||
setMessage(null)
|
||||
|
||||
try {
|
||||
const result = await apiService.restoreDatabase(restoreFile)
|
||||
setMessage({ type: 'success', text: result.message })
|
||||
setRestoreFile(null)
|
||||
} catch (err: any) {
|
||||
setMessage({
|
||||
type: 'error',
|
||||
text: err?.response?.data?.details || err?.response?.data?.error || 'Error al restaurar la base de datos',
|
||||
})
|
||||
} finally {
|
||||
setRestoring(false)
|
||||
}
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="p-6">
|
||||
<h1 className="text-2xl font-bold mb-6 flex items-center gap-2">
|
||||
|
|
@ -42,6 +69,16 @@ export function AdminSettings() {
|
|||
Configuración del Sistema
|
||||
</h1>
|
||||
|
||||
{message && (
|
||||
<div className={`max-w-2xl mb-6 p-3 rounded-md text-sm ${
|
||||
message.type === 'success'
|
||||
? 'bg-green-100 text-green-800 dark:bg-green-900/30 dark:text-green-200'
|
||||
: 'bg-red-100 text-red-800 dark:bg-red-900/30 dark:text-red-200'
|
||||
}`}>
|
||||
{message.text}
|
||||
</div>
|
||||
)}
|
||||
|
||||
<div className="max-w-2xl space-y-6">
|
||||
<div className="bg-white dark:bg-gray-800 rounded-lg shadow p-6">
|
||||
<h2 className="text-lg font-semibold mb-4 flex items-center gap-2">
|
||||
|
|
@ -75,16 +112,6 @@ export function AdminSettings() {
|
|||
</>
|
||||
)}
|
||||
</button>
|
||||
|
||||
{message && (
|
||||
<div className={`mt-4 p-3 rounded-md text-sm ${
|
||||
message.type === 'success'
|
||||
? 'bg-green-100 text-green-800 dark:bg-green-900/30 dark:text-green-200'
|
||||
: 'bg-red-100 text-red-800 dark:bg-red-900/30 dark:text-red-200'
|
||||
}`}>
|
||||
{message.text}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
|
@ -112,6 +139,47 @@ export function AdminSettings() {
|
|||
Descargar Backup .zip
|
||||
</button>
|
||||
</div>
|
||||
|
||||
<div className="p-4 border border-gray-100 dark:border-gray-700 rounded-lg bg-gray-50 dark:bg-gray-900/50">
|
||||
<h3 className="font-medium mb-1 flex items-center gap-2">
|
||||
<Upload className="h-4 w-4 text-blue-600" />
|
||||
Restaurar Base de Datos
|
||||
</h3>
|
||||
<p className="text-sm text-gray-500 mb-4">
|
||||
Sube un archivo de copia de seguridad (.sql o .zip) para restaurar la base de datos.
|
||||
La restauración sobrescribe los datos existentes. Se recomienda restaurar después de un reset.
|
||||
</p>
|
||||
<div className="flex items-center gap-3 flex-wrap">
|
||||
<input
|
||||
type="file"
|
||||
accept=".sql,.zip"
|
||||
onChange={(e) => setRestoreFile(e.target.files?.[0] ?? null)}
|
||||
className="block w-full max-w-sm text-sm text-gray-500 dark:text-gray-400 file:mr-4 file:py-2 file:px-4 file:rounded-md file:border-0 file:bg-blue-600 file:text-white file:cursor-pointer hover:file:bg-blue-700"
|
||||
/>
|
||||
<button
|
||||
onClick={handleRestore}
|
||||
disabled={restoring || !restoreFile}
|
||||
className="flex items-center gap-2 px-4 py-2 bg-blue-600 text-white rounded-md hover:bg-blue-700 disabled:opacity-50 disabled:cursor-not-allowed"
|
||||
>
|
||||
{restoring ? (
|
||||
<>
|
||||
<RefreshCw className="h-4 w-4 animate-spin" />
|
||||
Restaurando...
|
||||
</>
|
||||
) : (
|
||||
<>
|
||||
<Upload className="h-4 w-4" />
|
||||
Restaurar Base de Datos
|
||||
</>
|
||||
)}
|
||||
</button>
|
||||
</div>
|
||||
{restoreFile && (
|
||||
<p className="mt-2 text-sm text-gray-500 dark:text-gray-400">
|
||||
Archivo seleccionado: {restoreFile.name}
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
|
|
|||
|
|
@ -132,9 +132,7 @@ export const apiService = {
|
|||
importFeeds: async (file: File): Promise<{ imported: number; skipped: number; failed: number; message: string }> => {
|
||||
const formData = new FormData()
|
||||
formData.append('file', file)
|
||||
const { data } = await api.post('/feeds/import', formData, {
|
||||
headers: { 'Content-Type': 'multipart/form-data' },
|
||||
})
|
||||
const { data } = await api.post('/feeds/import', formData)
|
||||
return data
|
||||
},
|
||||
|
||||
|
|
@ -192,4 +190,11 @@ export const apiService = {
|
|||
alert('Error al descargar la copia de seguridad. Verifica tu conexión o permisos.');
|
||||
}
|
||||
},
|
||||
|
||||
restoreDatabase: async (file: File): Promise<{ message: string; filename: string; output?: string }> => {
|
||||
const formData = new FormData()
|
||||
formData.append('file', file)
|
||||
const { data } = await api.post('/admin/restore', formData)
|
||||
return data
|
||||
},
|
||||
}
|
||||
|
|
|
|||
11
init-db/36-add_eventos_cluster_columns.sql
Normal file
11
init-db/36-add_eventos_cluster_columns.sql
Normal file
|
|
@ -0,0 +1,11 @@
|
|||
BEGIN;
|
||||
|
||||
-- Columnas para el clustering de noticias (cluster_worker.py)
|
||||
ALTER TABLE eventos
|
||||
ADD COLUMN IF NOT EXISTS centroid JSONB,
|
||||
ADD COLUMN IF NOT EXISTS total_traducciones INTEGER DEFAULT 1,
|
||||
ADD COLUMN IF NOT EXISTS fecha_inicio TIMESTAMP,
|
||||
ADD COLUMN IF NOT EXISTS fecha_fin TIMESTAMP,
|
||||
ADD COLUMN IF NOT EXISTS n_noticias INTEGER DEFAULT 1;
|
||||
|
||||
COMMIT;
|
||||
18
init-db/37-topics-seed.sql
Normal file
18
init-db/37-topics-seed.sql
Normal file
|
|
@ -0,0 +1,18 @@
|
|||
BEGIN;
|
||||
|
||||
INSERT INTO topics (nombre, descripcion, weight, keywords) VALUES
|
||||
('Política', 'Noticias políticas y de gobierno', 3, 'política,político,gobierno,presidente,ministro,parlamento,elecciones,partido,senado,diputado'),
|
||||
('Economía', 'Economía y finanzas', 3, 'economía,económico,finanzas,mercado,inflación,banco,bancos,empresa,empresas,negocio,bolsa,inversión,presupuesto,impuestos'),
|
||||
('Seguridad', 'Seguridad y defensa', 3, 'seguridad,ataque,ataques,explosión,terrorismo,terrorista,ejército,militar,patrulla,armas,policía'),
|
||||
('Afganistán', 'Asuntos de Afganistán', 4, 'afganistán,afgano,afgana,kabul,candahar,kandahar,herat,mazar,talibán,talibanes,emirato'),
|
||||
('Conflicto', 'Conflictos y guerra', 3, 'conflicto,guerra,enfrentamiento,enfrentamientos,crisis,víctimas,frontera,invasIón,ocupación'),
|
||||
('Internacional', 'Relaciones internacionales', 2, 'internacional,diplomacia,diplomático,acuerdo,acuerdos,naciones unidas,embajador,tratado,extranjero'),
|
||||
('Deportes', 'Deportes y competiciones', 2, 'deportes,deporte,fútbol,futbol,partido,equipo,liga,campeonato,tenis,baloncesto,cricket,selección'),
|
||||
('Salud', 'Salud y medicina', 2, 'salud,hospital,médico,médicos,enfermedad,vacuna,tratamiento,pandemia,clínica,medicina'),
|
||||
('Tecnología', 'Tecnología y digital', 2, 'tecnología,internet,digital,telefonía,telecomunicaciones,software,aplicación,aplicaciones,internet,tecnológico'),
|
||||
('Educación', 'Educación y enseñanza', 2, 'educación,escuela,escuelas,universidad,universidades,estudiantes,profesor,enseñanza,academia,alumnos'),
|
||||
('Cultura', 'Cultura y arte', 1, 'cultura,arte,literatura,cine,música,música,patrimonio,historia,museo,libro,libros'),
|
||||
('Medio Ambiente', 'Clima y medio ambiente', 2, 'medio ambiente,clima,cambio climático,contaminación,sequía,inundación,agua,ecología,desastre natural')
|
||||
ON CONFLICT (nombre) DO NOTHING;
|
||||
|
||||
COMMIT;
|
||||
7
init-db/38-add_news_topics_score.sql
Normal file
7
init-db/38-add_news_topics_score.sql
Normal file
|
|
@ -0,0 +1,7 @@
|
|||
BEGIN;
|
||||
|
||||
-- El worker topics escribe en score (topics/main.go usa news_topics.score)
|
||||
ALTER TABLE news_topics
|
||||
ADD COLUMN IF NOT EXISTS score INTEGER DEFAULT 0;
|
||||
|
||||
COMMIT;
|
||||
10
init-db/39-add_related_traduccion_id.sql
Normal file
10
init-db/39-add_related_traduccion_id.sql
Normal file
|
|
@ -0,0 +1,10 @@
|
|||
BEGIN;
|
||||
|
||||
-- El worker related (cmd/related/main.go) trabaja con traducciones
|
||||
ALTER TABLE related_noticias
|
||||
ADD COLUMN IF NOT EXISTS related_traduccion_id INTEGER REFERENCES traducciones(id) ON DELETE CASCADE;
|
||||
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS idx_related_tr_pair
|
||||
ON related_noticias(traduccion_id, related_traduccion_id);
|
||||
|
||||
COMMIT;
|
||||
2
init-db/40-add_vectorization_columns.sql
Normal file
2
init-db/40-add_vectorization_columns.sql
Normal file
|
|
@ -0,0 +1,2 @@
|
|||
ALTER TABLE traducciones ADD COLUMN IF NOT EXISTS vectorization_date TIMESTAMP;
|
||||
ALTER TABLE traducciones ADD COLUMN IF NOT EXISTS qdrant_point_id TEXT;
|
||||
|
|
@ -73,6 +73,8 @@ CT2_MODEL_PATH = _env_str("CT2_MODEL_PATH", "/app/models/nllb-ct2")
|
|||
CT2_DEVICE = _env_str("CT2_DEVICE", "cpu")
|
||||
CT2_COMPUTE_TYPE = _env_str("CT2_COMPUTE_TYPE", "int8")
|
||||
UNIVERSAL_MODEL = _env_str("UNIVERSAL_MODEL", "facebook/nllb-200-distilled-600M")
|
||||
CT2_INTRA_THREADS = _env_int("CT2_INTRA_THREADS", 0)
|
||||
CT2_INTER_THREADS = _env_int("CT2_INTER_THREADS", 1)
|
||||
BODY_CHARS_CHUNK = _env_int("BODY_CHARS_CHUNK", 900)
|
||||
|
||||
LANG_CODE_MAP = {
|
||||
|
|
@ -128,7 +130,7 @@ def ensure_model():
|
|||
all_files_ok = True
|
||||
for f in required_files:
|
||||
fpath = os.path.join(model_path, f)
|
||||
if not os.path.exists(fpath) or os.path.getsize(fpath) < 1000:
|
||||
if not os.path.exists(fpath) or os.path.getsize(fpath) < 100:
|
||||
all_files_ok = False
|
||||
break
|
||||
|
||||
|
|
@ -157,6 +159,8 @@ def ensure_model():
|
|||
model_path,
|
||||
device=device,
|
||||
compute_type=CT2_COMPUTE_TYPE,
|
||||
inter_threads=CT2_INTER_THREADS,
|
||||
intra_threads=CT2_INTRA_THREADS,
|
||||
)
|
||||
|
||||
_tokenizer = AutoTokenizer.from_pretrained(UNIVERSAL_MODEL)
|
||||
|
|
@ -314,11 +318,7 @@ def translate_body_long(src: str, tgt: str, body: str) -> str:
|
|||
if len(chunks) == 1:
|
||||
return translate_texts(src, tgt, [body])[0]
|
||||
|
||||
translated_chunks = []
|
||||
for ch in chunks:
|
||||
tr = translate_texts(src, tgt, [ch])[0]
|
||||
translated_chunks.append(tr)
|
||||
|
||||
translated_chunks = translate_texts(src, tgt, chunks)
|
||||
return " ".join(translated_chunks)
|
||||
|
||||
|
||||
|
|
@ -406,18 +406,34 @@ def process_batch(conn, rows):
|
|||
titles = [i["titulo"] for i in items]
|
||||
translated_titles = translate_texts(lang_from, lang_to, titles)
|
||||
|
||||
for item, tt in zip(items, translated_titles):
|
||||
# Collect all body chunks across all items for a single batched call
|
||||
flat_chunks = []
|
||||
flat_keys = []
|
||||
for item in items:
|
||||
body = (item["resumen"] or "").strip()
|
||||
tb = ""
|
||||
if body:
|
||||
try:
|
||||
tb = translate_body_long(lang_from, lang_to, body)
|
||||
except Exception as e:
|
||||
LOG.error(f"Body translation error for ID {item['tr_id']}: {e}")
|
||||
tb = item["resumen"]
|
||||
chunks = split_body_into_chunks(body)
|
||||
flat_chunks.extend(chunks)
|
||||
flat_keys.extend([(item["tr_id"], i) for i in range(len(chunks))])
|
||||
|
||||
tt = clean_text((tt or "").strip())
|
||||
tb = clean_text((tb or "").strip())
|
||||
translated_bodies = []
|
||||
if flat_chunks:
|
||||
try:
|
||||
translated_bodies = translate_texts(lang_from, lang_to, flat_chunks)
|
||||
except Exception as e:
|
||||
LOG.error(f"Batch body translation error: {e}")
|
||||
translated_bodies = flat_chunks
|
||||
|
||||
body_parts = defaultdict(list)
|
||||
for (tr_id, _), tr in zip(flat_keys, translated_bodies):
|
||||
if tr is None:
|
||||
continue
|
||||
body_parts[tr_id].append(tr)
|
||||
|
||||
for idx, item in enumerate(items):
|
||||
tt = clean_text((translated_titles[idx] or "").strip())
|
||||
parts = body_parts.get(item["tr_id"])
|
||||
tb = clean_text(" ".join(parts).strip()) if parts else ""
|
||||
|
||||
if not tt:
|
||||
tt = item["titulo"]
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue