fix(backup): corregir descarga y añadir restauración de BD
- Backend usa DATABASE_URL (resolvePgConnInfo) en vez de DB_* que no existían en el contenedor API, arreglando el fallo de pg_dump - Dockerfile backend: alpine 3.23 para pg_dump 18.4 (compatible con PostgreSQL 18 del servidor) - BackupDatabase ya no usa docker exec (no disponible en la imagen) - Nuevo endpoint POST /api/admin/restore para restaurar .sql o .zip vía psql - UI de Configuración: sección Restaurar Base de Datos - Regenerado go.sum para el build
This commit is contained in:
parent
ecd7a3cdf8
commit
a285021e6c
22 changed files with 7624 additions and 450 deletions
|
|
@ -12,7 +12,7 @@ RUN go mod tidy
|
|||
|
||||
RUN CGO_ENABLED=0 GOOS=linux go build -buildvcs=false -o /server ./cmd/server
|
||||
|
||||
FROM alpine:3.19
|
||||
FROM alpine:3.23
|
||||
|
||||
RUN apk add --no-cache ca-certificates tzdata postgresql-client
|
||||
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@ import (
|
|||
"os"
|
||||
"os/signal"
|
||||
"strconv"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
|
|
@ -23,7 +24,6 @@ var (
|
|||
logger *log.Logger
|
||||
dbPool *pgxpool.Pool
|
||||
qdrantURL string
|
||||
ollamaURL string
|
||||
collection = "news_vectors"
|
||||
sleepSec = 30
|
||||
batchSize = 100
|
||||
|
|
@ -39,7 +39,6 @@ func loadConfig() {
|
|||
qdrantHost := getEnv("QDRANT_HOST", "localhost")
|
||||
qdrantPort := getEnvInt("QDRANT_PORT", 6333)
|
||||
qdrantURL = fmt.Sprintf("http://%s:%d", qdrantHost, qdrantPort)
|
||||
ollamaURL = getEnv("OLLAMA_URL", "http://ollama:11434")
|
||||
collection = getEnv("QDRANT_COLLECTION", "news_vectors")
|
||||
}
|
||||
|
||||
|
|
@ -61,7 +60,7 @@ func getEnvInt(key string, defaultValue int) int {
|
|||
|
||||
type Translation struct {
|
||||
ID int64
|
||||
NoticiaID int64
|
||||
NoticiaID string
|
||||
Lang string
|
||||
Titulo string
|
||||
Resumen string
|
||||
|
|
@ -70,6 +69,7 @@ type Translation struct {
|
|||
FuenteNombre string
|
||||
CategoriaID *int64
|
||||
PaisID *int64
|
||||
Embedding []float64
|
||||
}
|
||||
|
||||
func getPendingTranslations(ctx context.Context) ([]Translation, error) {
|
||||
|
|
@ -84,9 +84,11 @@ func getPendingTranslations(ctx context.Context) ([]Translation, error) {
|
|||
n.fecha,
|
||||
n.fuente_nombre,
|
||||
n.categoria_id,
|
||||
n.pais_id
|
||||
n.pais_id,
|
||||
te.embedding::text
|
||||
FROM traducciones t
|
||||
INNER JOIN noticias n ON t.noticia_id = n.id
|
||||
INNER JOIN traduccion_embeddings te ON te.traduccion_id = t.id
|
||||
WHERE t.vectorized = FALSE
|
||||
AND t.status = 'done'
|
||||
ORDER BY t.created_at ASC
|
||||
|
|
@ -100,54 +102,41 @@ func getPendingTranslations(ctx context.Context) ([]Translation, error) {
|
|||
var translations []Translation
|
||||
for rows.Next() {
|
||||
var t Translation
|
||||
var embText string
|
||||
if err := rows.Scan(
|
||||
&t.ID, &t.NoticiaID, &t.Lang, &t.Titulo, &t.Resumen,
|
||||
&t.URL, &t.Fecha, &t.FuenteNombre, &t.CategoriaID, &t.PaisID,
|
||||
&embText,
|
||||
); err != nil {
|
||||
logger.Printf("Error scanning translation row: %v", err)
|
||||
continue
|
||||
}
|
||||
emb, err := parseEmbedding(embText)
|
||||
if err != nil {
|
||||
logger.Printf("Error parsing embedding for translation %d: %v", t.ID, err)
|
||||
continue
|
||||
}
|
||||
t.Embedding = emb
|
||||
translations = append(translations, t)
|
||||
}
|
||||
return translations, nil
|
||||
}
|
||||
|
||||
type EmbeddingRequest struct {
|
||||
Model string `json:"model"`
|
||||
Input string `json:"input"`
|
||||
}
|
||||
|
||||
type EmbeddingResponse struct {
|
||||
Embedding []float64 `json:"embedding"`
|
||||
}
|
||||
|
||||
func generateEmbedding(text string) ([]float64, error) {
|
||||
reqBody := EmbeddingRequest{
|
||||
Model: "mxbai-embed-large",
|
||||
Input: text,
|
||||
func parseEmbedding(s string) ([]float64, error) {
|
||||
s = strings.Trim(strings.TrimSpace(s), "{}")
|
||||
if s == "" {
|
||||
return nil, fmt.Errorf("empty embedding")
|
||||
}
|
||||
|
||||
body, err := json.Marshal(reqBody)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
parts := strings.Split(s, ",")
|
||||
emb := make([]float64, len(parts))
|
||||
for i, p := range parts {
|
||||
v, err := strconv.ParseFloat(strings.TrimSpace(p), 64)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
emb[i] = v
|
||||
}
|
||||
|
||||
client := &http.Client{Timeout: 60 * time.Second}
|
||||
resp, err := client.Post(ollamaURL+"/api/embeddings", "application/json", bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, fmt.Errorf("Ollama returned status %d", resp.StatusCode)
|
||||
}
|
||||
|
||||
var result EmbeddingResponse
|
||||
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return result.Embedding, nil
|
||||
return emb, nil
|
||||
}
|
||||
|
||||
type QdrantPoint struct {
|
||||
|
|
@ -160,7 +149,7 @@ type QdrantUpsertRequest struct {
|
|||
Points []QdrantPoint `json:"points"`
|
||||
}
|
||||
|
||||
func ensureCollection() error {
|
||||
func ensureCollection(ctx context.Context) error {
|
||||
req, err := http.NewRequest("GET", qdrantURL+"/collections/"+collection, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
|
|
@ -177,16 +166,15 @@ func ensureCollection() error {
|
|||
return nil
|
||||
}
|
||||
|
||||
// Get embedding dimension
|
||||
emb, err := generateEmbedding("test")
|
||||
// Get embedding dimension from stored embeddings
|
||||
var dimension int
|
||||
err = dbPool.QueryRow(ctx, `SELECT dim FROM traduccion_embeddings LIMIT 1`).Scan(&dimension)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get embedding dimension: %w", err)
|
||||
}
|
||||
dimension := len(emb)
|
||||
|
||||
// Create collection
|
||||
createReq := map[string]interface{}{
|
||||
"name": collection,
|
||||
"vectors": map[string]interface{}{
|
||||
"size": dimension,
|
||||
"distance": "Cosine",
|
||||
|
|
@ -194,26 +182,35 @@ func ensureCollection() error {
|
|||
}
|
||||
|
||||
body, _ := json.Marshal(createReq)
|
||||
resp2, err := http.Post(qdrantURL+"/collections", "application/json", bytes.NewReader(body))
|
||||
createURL := qdrantURL + "/collections/" + collection
|
||||
req2, err := http.NewRequest(http.MethodPut, createURL, bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req2.Header.Set("Content-Type", "application/json")
|
||||
resp2, err := http.DefaultClient.Do(req2)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp2.Body.Close()
|
||||
|
||||
if resp2.StatusCode != http.StatusOK {
|
||||
respBody, _ := io.ReadAll(resp2.Body)
|
||||
return fmt.Errorf("Qdrant create collection returned %d: %s", resp2.StatusCode, string(respBody))
|
||||
}
|
||||
|
||||
logger.Printf("Created collection %s with dimension %d", collection, dimension)
|
||||
return nil
|
||||
}
|
||||
|
||||
func uploadToQdrant(translations []Translation, embeddings [][]float64) error {
|
||||
func uploadToQdrant(translations []Translation, pointIDs []string) error {
|
||||
points := make([]QdrantPoint, 0, len(translations))
|
||||
|
||||
for i, t := range translations {
|
||||
if embeddings[i] == nil {
|
||||
if t.Embedding == nil || len(t.Embedding) == 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
pointID := uuid.New().String()
|
||||
|
||||
payload := map[string]interface{}{
|
||||
"news_id": t.NoticiaID,
|
||||
"traduccion_id": t.ID,
|
||||
|
|
@ -235,8 +232,8 @@ func uploadToQdrant(translations []Translation, embeddings [][]float64) error {
|
|||
}
|
||||
|
||||
points = append(points, QdrantPoint{
|
||||
ID: pointID,
|
||||
Vector: embeddings[i],
|
||||
ID: pointIDs[i],
|
||||
Vector: t.Embedding,
|
||||
Payload: payload,
|
||||
})
|
||||
}
|
||||
|
|
@ -252,7 +249,12 @@ func uploadToQdrant(translations []Translation, embeddings [][]float64) error {
|
|||
}
|
||||
|
||||
url := fmt.Sprintf("%s/collections/%s/points", qdrantURL, collection)
|
||||
resp, err := http.Post(url, "application/json", bytes.NewReader(body))
|
||||
req, err := http.NewRequest(http.MethodPut, url, bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
@ -317,7 +319,7 @@ func main() {
|
|||
|
||||
ctx := context.Background()
|
||||
|
||||
if err := ensureCollection(); err != nil {
|
||||
if err := ensureCollection(ctx); err != nil {
|
||||
logger.Printf("Warning: Could not ensure collection: %v", err)
|
||||
}
|
||||
|
||||
|
|
@ -330,8 +332,8 @@ func main() {
|
|||
os.Exit(0)
|
||||
}()
|
||||
|
||||
logger.Printf("Config: qdrant=%s, ollama=%s, collection=%s, sleep=%ds, batch=%d",
|
||||
qdrantURL, ollamaURL, collection, sleepSec, batchSize)
|
||||
logger.Printf("Config: qdrant=%s, collection=%s, sleep=%ds, batch=%d",
|
||||
qdrantURL, collection, sleepSec, batchSize)
|
||||
|
||||
totalProcessed := 0
|
||||
|
||||
|
|
@ -351,30 +353,19 @@ func main() {
|
|||
|
||||
logger.Printf("Processing %d translations...", len(translations))
|
||||
|
||||
// Generate embeddings
|
||||
embeddings := make([][]float64, len(translations))
|
||||
for i, t := range translations {
|
||||
text := fmt.Sprintf("%s %s", t.Titulo, t.Resumen)
|
||||
emb, err := generateEmbedding(text)
|
||||
if err != nil {
|
||||
logger.Printf("Error generating embedding for %d: %v", t.ID, err)
|
||||
continue
|
||||
}
|
||||
embeddings[i] = emb
|
||||
}
|
||||
|
||||
// Upload to Qdrant
|
||||
if err := uploadToQdrant(translations, embeddings); err != nil {
|
||||
logger.Printf("Error uploading to Qdrant: %v", err)
|
||||
continue
|
||||
}
|
||||
|
||||
// Update DB status
|
||||
// Generate point IDs once (used both in Qdrant and DB)
|
||||
pointIDs := make([]string, len(translations))
|
||||
for i := range translations {
|
||||
pointIDs[i] = uuid.New().String()
|
||||
}
|
||||
|
||||
// Upload to Qdrant
|
||||
if err := uploadToQdrant(translations, pointIDs); err != nil {
|
||||
logger.Printf("Error uploading to Qdrant: %v", err)
|
||||
continue
|
||||
}
|
||||
|
||||
// Update DB status
|
||||
if err := updateTranslationStatus(ctx, translations, pointIDs); err != nil {
|
||||
logger.Printf("Error updating status: %v", err)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -187,6 +187,7 @@ func main() {
|
|||
admin.POST("/entities/retype", handlers.PatchEntityTipo)
|
||||
admin.GET("/backup", handlers.BackupDatabase)
|
||||
admin.GET("/backup/news", handlers.BackupNewsZipped)
|
||||
admin.POST("/restore", handlers.RestoreDatabase)
|
||||
admin.GET("/users", handlers.GetUsers)
|
||||
admin.POST("/users/:id/promote", handlers.PromoteUser)
|
||||
admin.POST("/users/:id/demote", handlers.DemoteUser)
|
||||
|
|
|
|||
|
|
@ -1,6 +1,8 @@
|
|||
module github.com/rss2/backend
|
||||
|
||||
go 1.22
|
||||
go 1.23
|
||||
|
||||
toolchain go1.23.12
|
||||
|
||||
require (
|
||||
github.com/PuerkitoBio/goquery v1.9.2
|
||||
|
|
|
|||
158
backend/go.sum
Normal file
158
backend/go.sum
Normal file
|
|
@ -0,0 +1,158 @@
|
|||
github.com/PuerkitoBio/goquery v1.9.2 h1:4/wZksC3KgkQw7SQgkKotmKljk0M6V8TUvA8Wb4yPeE=
|
||||
github.com/PuerkitoBio/goquery v1.9.2/go.mod h1:GHPCaP0ODyyxqcNoFGYlAprUFH81NuRPd0GX3Zu2Mvk=
|
||||
github.com/andybalholm/cascadia v1.3.2 h1:3Xi6Dw5lHF15JtdcmAHD3i1+T8plmv7BQ/nsViSLyss=
|
||||
github.com/andybalholm/cascadia v1.3.2/go.mod h1:7gtRlve5FxPPgIgX36uWBX58OdBsSS6lUvCFb+h7KvU=
|
||||
github.com/bsm/ginkgo/v2 v2.7.0 h1:ItPMPH90RbmZJt5GtkcNvIRuGEdwlBItdNVoyzaNQao=
|
||||
github.com/bsm/ginkgo/v2 v2.7.0/go.mod h1:AiKlXPm7ItEHNc/2+OkrNG4E0ITzojb9/xWzvQ9XZ9w=
|
||||
github.com/bsm/gomega v1.26.0 h1:LhQm+AFcgV2M0WyKroMASzAzCAJVpAxQXv4SaI9a69Y=
|
||||
github.com/bsm/gomega v1.26.0/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0=
|
||||
github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1O2AihPM=
|
||||
github.com/bytedance/sonic v1.9.1 h1:6iJ6NqdoxCDr6mbY8h18oSO+cShGSMRGCEo7F2h0x8s=
|
||||
github.com/bytedance/sonic v1.9.1/go.mod h1:i736AoUSYt75HyZLoJW9ERYxcy6eaN6h4BZXU064P/U=
|
||||
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
|
||||
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
github.com/chenzhuoyu/base64x v0.0.0-20211019084208-fb5309c8db06/go.mod h1:DH46F32mSOjUmXrMHnKwZdA8wcEefY7UVqBKYGjpdQY=
|
||||
github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 h1:qSGYFH7+jGhDF8vLC+iwCD4WpbV1EBDSzWkJODFLams=
|
||||
github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311/go.mod h1:b583jCggY9gE99b6G5LEC39OIiVsWj+R97kbl5odCEk=
|
||||
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
|
||||
github.com/gabriel-vasile/mimetype v1.4.2 h1:w5qFW6JKBz9Y393Y4q372O9A7cUSequkh1Q7OhCmWKU=
|
||||
github.com/gabriel-vasile/mimetype v1.4.2/go.mod h1:zApsH/mKG4w07erKIaJPFiX0Tsq9BFQgN3qGY5GnNgA=
|
||||
github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE=
|
||||
github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI=
|
||||
github.com/gin-gonic/gin v1.9.1 h1:4idEAncQnU5cB7BeOkPtxjfCSye0AAm1R0RVIqJ+Jmg=
|
||||
github.com/gin-gonic/gin v1.9.1/go.mod h1:hPrL7YrpYKXt5YId3A/Tnip5kqbEAP+KLuI3SUcPTeU=
|
||||
github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s=
|
||||
github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
|
||||
github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA=
|
||||
github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY=
|
||||
github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY=
|
||||
github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY=
|
||||
github.com/go-playground/validator/v10 v10.14.0 h1:vgvQWe3XCz3gIeFDm/HnTIbj6UGmg/+t63MyGU2n5js=
|
||||
github.com/go-playground/validator/v10 v10.14.0/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU=
|
||||
github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU=
|
||||
github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
|
||||
github.com/golang-jwt/jwt/v5 v5.0.0 h1:1n1XNM9hk7O9mnQoNBGolZvzebBQ7p93ULHRc28XJUE=
|
||||
github.com/golang-jwt/jwt/v5 v5.0.0/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk=
|
||||
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
|
||||
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
|
||||
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
|
||||
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/gorilla/websocket v1.5.1 h1:gmztn0JnHVt9JZquRuzLw3g4wouNVzKL15iLr/zn/QY=
|
||||
github.com/gorilla/websocket v1.5.1/go.mod h1:x3kM2JMyaluk02fnUJpQuwD2dCS5NDG2ZHL0uE0tcaY=
|
||||
github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
|
||||
github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
|
||||
github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk=
|
||||
github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=
|
||||
github.com/jackc/pgx/v5 v5.4.3 h1:cxFyXhxlvAifxnkKKdlxv8XqUf59tDlYjnV5YYfsJJY=
|
||||
github.com/jackc/pgx/v5 v5.4.3/go.mod h1:Ig06C2Vu0t5qXC60W8sqIthScaEnFvojjj9dSljmHRA=
|
||||
github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk=
|
||||
github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
|
||||
github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
|
||||
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
|
||||
github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
|
||||
github.com/klauspost/cpuid/v2 v2.2.4 h1:acbojRNwl3o09bUq+yDCtZFc1aiwaAAxtcn8YkZXnvk=
|
||||
github.com/klauspost/cpuid/v2 v2.2.4/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY=
|
||||
github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0=
|
||||
github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk=
|
||||
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
|
||||
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
|
||||
github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q=
|
||||
github.com/leodido/go-urn v1.2.4/go.mod h1:7ZrI8mTSeBSHl/UaRyKQW1qZeMgak41ANeCNaVckg+4=
|
||||
github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA=
|
||||
github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
|
||||
github.com/mmcdole/gofeed v1.2.1 h1:tPbFN+mfOLcM1kDF1x2c/N68ChbdBatkppdzf/vDe1s=
|
||||
github.com/mmcdole/gofeed v1.2.1/go.mod h1:2wVInNpgmC85q16QTTuwbuKxtKkHLCDDtf0dCmnrNr4=
|
||||
github.com/mmcdole/goxpp v1.1.0 h1:WwslZNF7KNAXTFuzRtn/OKZxFLJAAyOA9w82mDz2ZGI=
|
||||
github.com/mmcdole/goxpp v1.1.0/go.mod h1:v+25+lT2ViuQ7mVxcncQ8ch1URund48oH+jhjiwEgS8=
|
||||
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
|
||||
github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
|
||||
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
|
||||
github.com/pelletier/go-toml/v2 v2.0.8 h1:0ctb6s9mE31h0/lhu+J6OPmVeDxJn+kYnJc2jZR9tGQ=
|
||||
github.com/pelletier/go-toml/v2 v2.0.8/go.mod h1:vuYfssBdrU2XDZ9bYydBu6t+6a6PYNcZljzZR9VXg+4=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/redis/go-redis/v9 v9.0.5 h1:CuQcn5HIEeK7BgElubPP8CGtE0KakrnbBSTLjathl5o=
|
||||
github.com/redis/go-redis/v9 v9.0.5/go.mod h1:WqMKv5vnQbRuZstUwxQI195wHy+t4PuXDOjzMvcuQHk=
|
||||
github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
|
||||
github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
|
||||
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
|
||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
|
||||
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
|
||||
github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
|
||||
github.com/stretchr/testify v1.8.3 h1:RP3t2pwF7cMEbC1dqtB6poj3niw/9gnV4Cjg5oW5gtY=
|
||||
github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
|
||||
github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI=
|
||||
github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08=
|
||||
github.com/ugorji/go/codec v1.2.11 h1:BMaWp1Bb6fHwEtbplGBGJ498wD+LKlNSl25MjdZY4dU=
|
||||
github.com/ugorji/go/codec v1.2.11/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg=
|
||||
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
|
||||
golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8=
|
||||
golang.org/x/arch v0.3.0 h1:02VY4/ZcO/gBOH6PUaoiptASxtXU10jazRCP865E97k=
|
||||
golang.org/x/arch v0.3.0/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
|
||||
golang.org/x/crypto v0.26.0 h1:RrRspgV4mU+YwB4FYnuBoKsUapNIL5cohGAmSH3azsw=
|
||||
golang.org/x/crypto v0.26.0/go.mod h1:GY7jblb9wI+FOo5y8/S2oY4zWP07AkOJ4+jxCqdqn54=
|
||||
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
|
||||
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
|
||||
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
|
||||
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
|
||||
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
|
||||
golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns=
|
||||
golang.org/x/net v0.28.0 h1:a9JDOJc5GMUJ0+UDqmLT86WiEy7iWyIhz8gz8E4e5hE=
|
||||
golang.org/x/net v0.28.0/go.mod h1:yqtgsTWOOnlGLG9GFRrK3++bGOUEkNBoHZc8MEDWPNg=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ=
|
||||
golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo=
|
||||
golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
|
||||
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
|
||||
golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
|
||||
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
|
||||
golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
|
||||
golang.org/x/text v0.17.0 h1:XtiM5bkSOt+ewxlOE/aE/AKEHibwj/6gvWMl9Rsh0Qc=
|
||||
golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
|
||||
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
|
||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg=
|
||||
google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4=
|
||||
|
|
@ -6,14 +6,15 @@ import (
|
|||
"context"
|
||||
"encoding/csv"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"os/exec"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/rss2/backend/internal/config"
|
||||
"github.com/rss2/backend/internal/db"
|
||||
|
|
@ -637,17 +638,73 @@ func PatchEntityTipo(c *gin.Context) {
|
|||
})
|
||||
}
|
||||
|
||||
// pgConnInfo holds PostgreSQL connection parameters for pg_dump
|
||||
type pgConnInfo struct {
|
||||
host, port, name, user, pass string
|
||||
}
|
||||
|
||||
// resolvePgConnInfo prefers DATABASE_URL (the variable the API container gets)
|
||||
// and falls back to the individual DB_* variables used by the workers.
|
||||
func resolvePgConnInfo() pgConnInfo {
|
||||
if dbURL := os.Getenv("DATABASE_URL"); dbURL != "" {
|
||||
if u, err := url.Parse(dbURL); err == nil && u.Hostname() != "" {
|
||||
info := pgConnInfo{
|
||||
host: u.Hostname(),
|
||||
user: u.User.Username(),
|
||||
name: strings.TrimPrefix(u.Path, "/"),
|
||||
port: u.Port(),
|
||||
}
|
||||
if p, ok := u.User.Password(); ok {
|
||||
info.pass = p
|
||||
}
|
||||
if info.port == "" {
|
||||
info.port = "5432"
|
||||
}
|
||||
if info.user == "" {
|
||||
info.user = "postgres"
|
||||
}
|
||||
if info.name == "" {
|
||||
info.name = "postgres"
|
||||
}
|
||||
return info
|
||||
}
|
||||
}
|
||||
|
||||
info := pgConnInfo{
|
||||
host: os.Getenv("DB_HOST"),
|
||||
port: os.Getenv("DB_PORT"),
|
||||
name: os.Getenv("DB_NAME"),
|
||||
user: os.Getenv("DB_USER"),
|
||||
pass: os.Getenv("DB_PASS"),
|
||||
}
|
||||
if info.host == "" {
|
||||
info.host = "db"
|
||||
}
|
||||
if info.port == "" {
|
||||
info.port = "5432"
|
||||
}
|
||||
if info.name == "" {
|
||||
info.name = "rss"
|
||||
}
|
||||
if info.user == "" {
|
||||
info.user = "rss"
|
||||
}
|
||||
return info
|
||||
}
|
||||
|
||||
// BackupDatabase runs pg_dump and returns the SQL as a downloadable file
|
||||
func BackupDatabase(c *gin.Context) {
|
||||
// Ejecutar pg_dump desde dentro del contenedor db para evitar problemas de red
|
||||
// Usamos docker exec para ejecutar el comando dentro del servicio db
|
||||
cmd := exec.Command("docker", "exec", "rss2_db", "pg_dump",
|
||||
"-U", os.Getenv("POSTGRES_USER"),
|
||||
"-d", os.Getenv("POSTGRES_DB"),
|
||||
info := resolvePgConnInfo()
|
||||
|
||||
cmd := exec.Command("pg_dump",
|
||||
"-h", info.host,
|
||||
"-p", info.port,
|
||||
"-U", info.user,
|
||||
"-d", info.name,
|
||||
"--no-password",
|
||||
"--format=plain",
|
||||
"--pghost=5432",
|
||||
)
|
||||
cmd.Env = append(os.Environ(), fmt.Sprintf("PGPASSWORD=%s", info.pass))
|
||||
|
||||
var out bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
|
|
@ -671,32 +728,16 @@ func BackupDatabase(c *gin.Context) {
|
|||
|
||||
// BackupNewsZipped performs a pg_dump of news tables and returns a ZIP file
|
||||
func BackupNewsZipped(c *gin.Context) {
|
||||
dbHost := os.Getenv("DB_HOST")
|
||||
if dbHost == "" {
|
||||
dbHost = "db"
|
||||
}
|
||||
dbPort := os.Getenv("DB_PORT")
|
||||
if dbPort == "" {
|
||||
dbPort = "5432"
|
||||
}
|
||||
dbName := os.Getenv("DB_NAME")
|
||||
if dbName == "" {
|
||||
dbName = "rss"
|
||||
}
|
||||
dbUser := os.Getenv("DB_USER")
|
||||
if dbUser == "" {
|
||||
dbUser = "rss"
|
||||
}
|
||||
dbPass := os.Getenv("DB_PASS")
|
||||
info := resolvePgConnInfo()
|
||||
|
||||
// Tables to backup
|
||||
tables := []string{"noticias", "traducciones", "tags", "tags_noticia"}
|
||||
|
||||
args := []string{
|
||||
"-h", dbHost,
|
||||
"-p", dbPort,
|
||||
"-U", dbUser,
|
||||
"-d", dbName,
|
||||
"-h", info.host,
|
||||
"-p", info.port,
|
||||
"-U", info.user,
|
||||
"-d", info.name,
|
||||
"--no-password",
|
||||
}
|
||||
|
||||
|
|
@ -705,7 +746,7 @@ func BackupNewsZipped(c *gin.Context) {
|
|||
}
|
||||
|
||||
cmd := exec.Command("pg_dump", args...)
|
||||
cmd.Env = append(os.Environ(), fmt.Sprintf("PGPASSWORD=%s", dbPass))
|
||||
cmd.Env = append(os.Environ(), fmt.Sprintf("PGPASSWORD=%s", info.pass))
|
||||
|
||||
var sqlOut bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
|
|
@ -748,3 +789,107 @@ func BackupNewsZipped(c *gin.Context) {
|
|||
c.Header("Cache-Control", "no-cache")
|
||||
c.Data(http.StatusOK, "application/zip", buf.Bytes())
|
||||
}
|
||||
|
||||
// RestoreDatabase accepts an uploaded .sql or .zip backup and restores it via psql
|
||||
func RestoreDatabase(c *gin.Context) {
|
||||
file, err := c.FormFile("file")
|
||||
if err != nil {
|
||||
c.JSON(http.StatusBadRequest, gin.H{
|
||||
"error": "No file uploaded",
|
||||
"message": "Se requiere un archivo .sql o .zip",
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
uploaded, err := file.Open()
|
||||
if err != nil {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": "Failed to open uploaded file"})
|
||||
return
|
||||
}
|
||||
defer uploaded.Close()
|
||||
|
||||
data, err := io.ReadAll(uploaded)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": "Failed to read uploaded file"})
|
||||
return
|
||||
}
|
||||
|
||||
var sqlData []byte
|
||||
lowerName := strings.ToLower(file.Filename)
|
||||
switch {
|
||||
case strings.HasSuffix(lowerName, ".sql"):
|
||||
sqlData = data
|
||||
case strings.HasSuffix(lowerName, ".zip"):
|
||||
zr, err := zip.NewReader(bytes.NewReader(data), int64(len(data)))
|
||||
if err != nil {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": "Invalid ZIP file", "message": err.Error()})
|
||||
return
|
||||
}
|
||||
found := false
|
||||
for _, zf := range zr.File {
|
||||
if zf.FileInfo().IsDir() || !strings.HasSuffix(strings.ToLower(zf.Name), ".sql") {
|
||||
continue
|
||||
}
|
||||
rc, err := zf.Open()
|
||||
if err != nil {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": "Failed to open SQL inside ZIP", "message": err.Error()})
|
||||
return
|
||||
}
|
||||
sqlData, err = io.ReadAll(rc)
|
||||
rc.Close()
|
||||
if err != nil {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": "Failed to read SQL inside ZIP", "message": err.Error()})
|
||||
return
|
||||
}
|
||||
found = true
|
||||
break
|
||||
}
|
||||
if !found {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": "No SQL file found inside ZIP"})
|
||||
return
|
||||
}
|
||||
default:
|
||||
c.JSON(http.StatusBadRequest, gin.H{
|
||||
"error": "Unsupported file type",
|
||||
"message": "Solo se permiten archivos .sql o .zip",
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
if len(sqlData) == 0 {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": "Empty backup file"})
|
||||
return
|
||||
}
|
||||
|
||||
info := resolvePgConnInfo()
|
||||
|
||||
cmd := exec.Command("psql",
|
||||
"-h", info.host,
|
||||
"-p", info.port,
|
||||
"-U", info.user,
|
||||
"-d", info.name,
|
||||
"--no-password",
|
||||
"-v", "ON_ERROR_STOP=1",
|
||||
)
|
||||
cmd.Env = append(os.Environ(), fmt.Sprintf("PGPASSWORD=%s", info.pass))
|
||||
cmd.Stdin = bytes.NewReader(sqlData)
|
||||
|
||||
var out bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
cmd.Stdout = &out
|
||||
cmd.Stderr = &stderr
|
||||
|
||||
if err := cmd.Run(); err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{
|
||||
"error": "Restore failed",
|
||||
"details": stderr.String(),
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
c.JSON(http.StatusOK, gin.H{
|
||||
"message": "Base de datos restaurada correctamente",
|
||||
"filename": file.Filename,
|
||||
"output": out.String(),
|
||||
})
|
||||
}
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ import (
|
|||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
|
|
@ -397,11 +398,46 @@ func ImportFeeds(c *gin.Context) {
|
|||
}
|
||||
|
||||
reader := csv.NewReader(strings.NewReader(string(content)))
|
||||
_, err = reader.Read()
|
||||
records, err := reader.ReadAll()
|
||||
if err != nil {
|
||||
c.JSON(http.StatusBadRequest, models.ErrorResponse{Error: "Invalid CSV format"})
|
||||
return
|
||||
}
|
||||
if len(records) == 0 {
|
||||
c.JSON(http.StatusBadRequest, models.ErrorResponse{Error: "Empty CSV file"})
|
||||
return
|
||||
}
|
||||
|
||||
colIndex := func(header []string, names ...string) int {
|
||||
for i, h := range header {
|
||||
h = strings.ToLower(strings.TrimSpace(h))
|
||||
for _, n := range names {
|
||||
if h == n {
|
||||
return i
|
||||
}
|
||||
}
|
||||
}
|
||||
return -1
|
||||
}
|
||||
|
||||
header := records[0]
|
||||
isHeader := colIndex(header, "url", "link", "feed", "nombre", "name", "titulo") != -1
|
||||
|
||||
var dataRows [][]string
|
||||
iNombre, iDesc, iURL, iCat, iPais, iLang, iActivo, iFallos := -1, -1, -1, -1, -1, -1, -1, -1
|
||||
if isHeader {
|
||||
iNombre = colIndex(header, "nombre", "name", "titulo", "title")
|
||||
iDesc = colIndex(header, "descripcion", "description")
|
||||
iURL = colIndex(header, "url", "link", "feed")
|
||||
iCat = colIndex(header, "categoria_id", "category_id", "categoria", "category")
|
||||
iPais = colIndex(header, "pais_id", "country_id", "pais", "country")
|
||||
iLang = colIndex(header, "idioma", "language", "lang")
|
||||
iActivo = colIndex(header, "activo", "active", "enabled")
|
||||
iFallos = colIndex(header, "fallos", "failures", "fails")
|
||||
dataRows = records[1:]
|
||||
} else {
|
||||
dataRows = records
|
||||
}
|
||||
|
||||
imported := 0
|
||||
skipped := 0
|
||||
|
|
@ -415,71 +451,115 @@ func ImportFeeds(c *gin.Context) {
|
|||
}
|
||||
defer tx.Rollback(context.Background())
|
||||
|
||||
for {
|
||||
record, err := reader.Read()
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
failed++
|
||||
continue
|
||||
field := func(record []string, idx int) string {
|
||||
if idx < 0 || idx >= len(record) {
|
||||
return ""
|
||||
}
|
||||
return strings.TrimSpace(record[idx])
|
||||
}
|
||||
|
||||
if len(record) < 4 {
|
||||
for _, record := range dataRows {
|
||||
if len(record) == 0 {
|
||||
skipped++
|
||||
continue
|
||||
}
|
||||
|
||||
nombre := strings.TrimSpace(record[1])
|
||||
url := strings.TrimSpace(record[3])
|
||||
var nombre, url string
|
||||
if isHeader {
|
||||
nombre = field(record, iNombre)
|
||||
url = field(record, iURL)
|
||||
} else if len(record) == 1 {
|
||||
url = field(record, 0)
|
||||
} else if len(record) == 2 {
|
||||
a := field(record, 0)
|
||||
b := field(record, 1)
|
||||
if looksLikeURL(a) {
|
||||
url, nombre = a, b
|
||||
} else {
|
||||
nombre, url = a, b
|
||||
}
|
||||
} else {
|
||||
nombre = field(record, 1)
|
||||
url = field(record, 3)
|
||||
}
|
||||
|
||||
if nombre == "" || url == "" {
|
||||
if url == "" {
|
||||
skipped++
|
||||
continue
|
||||
}
|
||||
if nombre == "" {
|
||||
nombre = nombreFromURL(url)
|
||||
}
|
||||
|
||||
var descripcion *string
|
||||
if len(record) > 2 && strings.TrimSpace(record[2]) != "" {
|
||||
descripcionStr := strings.TrimSpace(record[2])
|
||||
descripcion = &descripcionStr
|
||||
}
|
||||
|
||||
var categoriaID *int64
|
||||
if len(record) > 4 && strings.TrimSpace(record[4]) != "" {
|
||||
catID, err := strconv.ParseInt(strings.TrimSpace(record[4]), 10, 64)
|
||||
if err == nil {
|
||||
categoriaID = &catID
|
||||
}
|
||||
}
|
||||
|
||||
var paisID *int64
|
||||
if len(record) > 6 && strings.TrimSpace(record[6]) != "" {
|
||||
pID, err := strconv.ParseInt(strings.TrimSpace(record[6]), 10, 64)
|
||||
if err == nil {
|
||||
paisID = &pID
|
||||
}
|
||||
}
|
||||
|
||||
var idioma *string
|
||||
if len(record) > 8 && strings.TrimSpace(record[8]) != "" {
|
||||
lang := strings.TrimSpace(record[8])
|
||||
if len(lang) > 2 {
|
||||
lang = lang[:2]
|
||||
}
|
||||
idioma = &lang
|
||||
}
|
||||
|
||||
activo := true
|
||||
if len(record) > 9 && strings.TrimSpace(record[9]) != "" {
|
||||
activo = strings.ToLower(strings.TrimSpace(record[9])) == "true"
|
||||
var fallos int64
|
||||
|
||||
if isHeader {
|
||||
if d := field(record, iDesc); d != "" {
|
||||
descripcion = &d
|
||||
}
|
||||
if v := field(record, iCat); v != "" {
|
||||
if id, err := strconv.ParseInt(v, 10, 64); err == nil {
|
||||
categoriaID = &id
|
||||
}
|
||||
}
|
||||
if v := field(record, iPais); v != "" {
|
||||
if id, err := strconv.ParseInt(v, 10, 64); err == nil {
|
||||
paisID = &id
|
||||
}
|
||||
}
|
||||
if l := field(record, iLang); l != "" {
|
||||
if len(l) > 2 {
|
||||
l = l[:2]
|
||||
}
|
||||
idioma = &l
|
||||
}
|
||||
if a := field(record, iActivo); a != "" {
|
||||
activo = strings.ToLower(a) == "true"
|
||||
}
|
||||
if f := field(record, iFallos); f != "" {
|
||||
if v, err := strconv.ParseInt(f, 10, 64); err == nil {
|
||||
fallos = v
|
||||
}
|
||||
}
|
||||
} else {
|
||||
if d := field(record, 2); d != "" {
|
||||
descripcion = &d
|
||||
}
|
||||
if v := field(record, 4); v != "" {
|
||||
if id, err := strconv.ParseInt(v, 10, 64); err == nil {
|
||||
categoriaID = &id
|
||||
}
|
||||
}
|
||||
if v := field(record, 6); v != "" {
|
||||
if id, err := strconv.ParseInt(v, 10, 64); err == nil {
|
||||
paisID = &id
|
||||
}
|
||||
}
|
||||
if l := field(record, 8); l != "" {
|
||||
if len(l) > 2 {
|
||||
l = l[:2]
|
||||
}
|
||||
idioma = &l
|
||||
}
|
||||
if a := field(record, 9); a != "" {
|
||||
activo = strings.ToLower(a) == "true"
|
||||
}
|
||||
if f := field(record, 10); f != "" {
|
||||
if v, err := strconv.ParseInt(f, 10, 64); err == nil {
|
||||
fallos = v
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var fallos int64
|
||||
if len(record) > 10 && strings.TrimSpace(record[10]) != "" {
|
||||
f, err := strconv.ParseInt(strings.TrimSpace(record[10]), 10, 64)
|
||||
if err == nil {
|
||||
fallos = f
|
||||
}
|
||||
if _, err := tx.Exec(context.Background(), "SAVEPOINT sp"); err != nil {
|
||||
failed++
|
||||
errors = append(errors, fmt.Sprintf("Error for %s: %v", url, err))
|
||||
continue
|
||||
}
|
||||
|
||||
var existingID int64
|
||||
|
|
@ -490,22 +570,25 @@ func ImportFeeds(c *gin.Context) {
|
|||
WHERE id=$8`,
|
||||
nombre, descripcion, categoriaID, paisID, idioma, activo, fallos, existingID,
|
||||
)
|
||||
if err != nil {
|
||||
failed++
|
||||
errors = append(errors, fmt.Sprintf("Error updating %s: %v", url, err))
|
||||
continue
|
||||
}
|
||||
} else {
|
||||
_, err = tx.Exec(context.Background(), `
|
||||
INSERT INTO feeds (nombre, descripcion, url, categoria_id, pais_id, idioma, activo, fallos)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8)`,
|
||||
nombre, descripcion, url, categoriaID, paisID, idioma, activo, fallos,
|
||||
)
|
||||
if err != nil {
|
||||
failed++
|
||||
errors = append(errors, fmt.Sprintf("Error inserting %s: %v", url, err))
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
tx.Exec(context.Background(), "ROLLBACK TO SAVEPOINT sp")
|
||||
failed++
|
||||
errors = append(errors, fmt.Sprintf("Error upserting %s: %v", url, err))
|
||||
continue
|
||||
}
|
||||
|
||||
if _, err := tx.Exec(context.Background(), "RELEASE SAVEPOINT sp"); err != nil {
|
||||
failed++
|
||||
errors = append(errors, fmt.Sprintf("Error for %s: %v", url, err))
|
||||
continue
|
||||
}
|
||||
|
||||
imported++
|
||||
|
|
@ -525,6 +608,22 @@ func ImportFeeds(c *gin.Context) {
|
|||
})
|
||||
}
|
||||
|
||||
func looksLikeURL(s string) bool {
|
||||
return strings.HasPrefix(s, "http://") || strings.HasPrefix(s, "https://") || strings.HasPrefix(s, "http://www.") || strings.HasPrefix(s, "https://www.") || strings.HasPrefix(s, "www.") || strings.HasPrefix(s, "feed://")
|
||||
}
|
||||
|
||||
func nombreFromURL(u string) string {
|
||||
u = strings.TrimSpace(u)
|
||||
if !strings.HasPrefix(u, "http://") && !strings.HasPrefix(u, "https://") {
|
||||
u = "https://" + u
|
||||
}
|
||||
parsed, err := url.Parse(u)
|
||||
if err != nil || parsed.Host == "" {
|
||||
return u
|
||||
}
|
||||
return strings.TrimPrefix(parsed.Host, "www.")
|
||||
}
|
||||
|
||||
func stringOrEmpty(s *string) string {
|
||||
if s == nil {
|
||||
return ""
|
||||
|
|
|
|||
|
|
@ -51,55 +51,45 @@ func SearchNews(c *gin.Context) {
|
|||
offset := (page - 1) * perPage
|
||||
|
||||
// Build dynamic query
|
||||
args := []interface{}{}
|
||||
argNum := 1
|
||||
args := []interface{}{lang}
|
||||
argNum := 2
|
||||
whereClause := "WHERE 1=1"
|
||||
|
||||
if query != "" {
|
||||
whereClause += " AND (n.titulo ILIKE $" + strconv.Itoa(argNum) + " OR n.resumen ILIKE $" + strconv.Itoa(argNum) + " OR n.contenido ILIKE $" + strconv.Itoa(argNum) + ")"
|
||||
whereClause += " AND (n.titulo ILIKE $" + strconv.Itoa(argNum) + " OR n.resumen ILIKE $" + strconv.Itoa(argNum) + ")"
|
||||
args = append(args, "%"+query+"%")
|
||||
argNum++
|
||||
}
|
||||
|
||||
if lang != "" {
|
||||
whereClause += " AND t.lang_to = $" + strconv.Itoa(argNum)
|
||||
args = append(args, lang)
|
||||
argNum++
|
||||
}
|
||||
|
||||
if categoriaID != "" {
|
||||
whereClause += " AND n.categoria_id = $" + strconv.Itoa(argNum)
|
||||
catID, err := strconv.ParseInt(categoriaID, 10, 64)
|
||||
if err == nil {
|
||||
if catID, err := strconv.ParseInt(categoriaID, 10, 64); err == nil {
|
||||
whereClause += " AND n.categoria_id = $" + strconv.Itoa(argNum)
|
||||
args = append(args, catID)
|
||||
argNum++
|
||||
}
|
||||
}
|
||||
|
||||
if paisID != "" {
|
||||
whereClause += " AND n.pais_id = $" + strconv.Itoa(argNum)
|
||||
pID, err := strconv.ParseInt(paisID, 10, 64)
|
||||
if err == nil {
|
||||
if pID, err := strconv.ParseInt(paisID, 10, 64); err == nil {
|
||||
whereClause += " AND n.pais_id = $" + strconv.Itoa(argNum)
|
||||
args = append(args, pID)
|
||||
argNum++
|
||||
}
|
||||
}
|
||||
|
||||
args = append(args, perPage, offset)
|
||||
|
||||
sqlQuery := `
|
||||
SELECT n.id, n.titulo, n.resumen, n.contenido, n.url, n.imagen,
|
||||
n.feed_id, n.lang, n.categoria_id, n.pais_id, n.created_at, n.updated_at,
|
||||
COALESCE(t.titulo_trad, '') as titulo_trad,
|
||||
COALESCE(t.resumen_trad, '') as resumen_trad,
|
||||
t.lang_to as lang_trad,
|
||||
f.nombre as fuente_nombre
|
||||
SELECT n.id, n.titulo, COALESCE(n.resumen, ''), n.url, n.fecha, n.imagen_url,
|
||||
n.categoria_id, n.pais_id, n.fuente_nombre,
|
||||
t.titulo_trad,
|
||||
t.resumen_trad,
|
||||
t.lang_to as lang_trad
|
||||
FROM noticias n
|
||||
LEFT JOIN traducciones t ON t.noticia_id = n.id AND t.lang_to = $` + strconv.Itoa(argNum) + `
|
||||
LEFT JOIN feeds f ON f.id = n.feed_id
|
||||
LEFT JOIN traducciones t ON t.noticia_id = n.id AND t.lang_to = $1
|
||||
` + whereClause + `
|
||||
ORDER BY n.created_at DESC
|
||||
LIMIT $` + strconv.Itoa(argNum+1) + ` OFFSET $` + strconv.Itoa(argNum+2)
|
||||
ORDER BY n.fecha DESC
|
||||
LIMIT $` + strconv.Itoa(argNum) + ` OFFSET $` + strconv.Itoa(argNum+1)
|
||||
|
||||
args = append(args, perPage, offset)
|
||||
|
||||
rows, err := db.GetPool().Query(ctx, sqlQuery, args...)
|
||||
if err != nil {
|
||||
|
|
@ -108,33 +98,38 @@ func SearchNews(c *gin.Context) {
|
|||
}
|
||||
defer rows.Close()
|
||||
|
||||
var newsList []models.NewsWithTranslations
|
||||
var newsList []NewsResponse
|
||||
for rows.Next() {
|
||||
var n models.NewsWithTranslations
|
||||
var imagen *string
|
||||
var n NewsResponse
|
||||
var imagenURL, fuenteNombre *string
|
||||
var categoriaIDp, paisIDp *int64
|
||||
|
||||
err := rows.Scan(
|
||||
&n.ID, &n.Title, &n.Summary, &n.Content, &n.URL, &imagen,
|
||||
&n.FeedID, &n.Lang, &n.CategoryID, &n.CountryID, &n.CreatedAt, &n.UpdatedAt,
|
||||
&n.ID, &n.Titulo, &n.Resumen, &n.URL, &n.Fecha, &imagenURL,
|
||||
&categoriaIDp, &paisIDp, &fuenteNombre,
|
||||
&n.TitleTranslated, &n.SummaryTranslated, &n.LangTranslated,
|
||||
)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if imagen != nil {
|
||||
n.ImageURL = imagen
|
||||
if imagenURL != nil {
|
||||
n.ImagenURL = imagenURL
|
||||
}
|
||||
if fuenteNombre != nil {
|
||||
n.FuenteNombre = *fuenteNombre
|
||||
}
|
||||
n.CategoriaID = categoriaIDp
|
||||
n.PaisID = paisIDp
|
||||
newsList = append(newsList, n)
|
||||
}
|
||||
|
||||
// Get total count
|
||||
countArgs := args[:len(args)-2]
|
||||
|
||||
// Remove LIMIT/OFFSET from args for count
|
||||
var total int
|
||||
err = db.GetPool().QueryRow(ctx, `
|
||||
SELECT COUNT(*) FROM noticias n
|
||||
LEFT JOIN traducciones t ON t.noticia_id = n.id AND t.lang_to = $`+strconv.Itoa(argNum)+`
|
||||
LEFT JOIN traducciones t ON t.noticia_id = n.id AND t.lang_to = $1
|
||||
`+whereClause, countArgs...).Scan(&total)
|
||||
if err != nil {
|
||||
total = len(newsList)
|
||||
|
|
@ -142,15 +137,13 @@ func SearchNews(c *gin.Context) {
|
|||
|
||||
totalPages := (total + perPage - 1) / perPage
|
||||
|
||||
response := models.NewsListResponse{
|
||||
News: newsList,
|
||||
Total: total,
|
||||
Page: page,
|
||||
PerPage: perPage,
|
||||
TotalPages: totalPages,
|
||||
}
|
||||
|
||||
c.JSON(http.StatusOK, response)
|
||||
c.JSON(http.StatusOK, gin.H{
|
||||
"news": newsList,
|
||||
"total": total,
|
||||
"page": page,
|
||||
"per_page": perPage,
|
||||
"total_pages": totalPages,
|
||||
})
|
||||
}
|
||||
|
||||
func GetStats(c *gin.Context) {
|
||||
|
|
|
|||
|
|
@ -132,7 +132,10 @@ func writeLoop(wsWorker *WSWorker) {
|
|||
for {
|
||||
select {
|
||||
case <-ticker.C:
|
||||
if err := wsWorker.conn.WriteMessage(websocket.PingMessage, nil); err != nil {
|
||||
wsWorker.mu.Lock()
|
||||
err := wsWorker.conn.WriteMessage(websocket.PingMessage, nil)
|
||||
wsWorker.mu.Unlock()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
|
|
@ -163,7 +166,9 @@ func heartbeatLoop(wsWorker *WSWorker) {
|
|||
|
||||
func sendWS(wsWorker *WSWorker, msg models.WSServerMessage) {
|
||||
data, _ := json.Marshal(msg)
|
||||
wsWorker.mu.Lock()
|
||||
wsWorker.conn.WriteMessage(websocket.TextMessage, data)
|
||||
wsWorker.mu.Unlock()
|
||||
}
|
||||
|
||||
func cleanupWorker(wsWorker *WSWorker, apiKey string) {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue