muchas cosas

backend/cmd/scraper/main.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"crypto/md5"
 	"fmt"
+	"io"
 	"log"
 	"net/http"
 	"os"
@@ -27,6 +28,7 @@ var (
 	batchSize     = 10
 	enrichLimit   = 20
 	scraperWorkers = 5
+	maxBodyBytes  = int64(512 << 10)
 )
 
 type URLSource struct {
@@ -169,7 +171,7 @@ func extractArticle(source URLSource) (*Article, error) {
 		return nil, fmt.Errorf("HTTP %d", resp.StatusCode)
 	}
 
-	doc, err := goquery.NewDocumentFromReader(resp.Body)
+	doc, err := goquery.NewDocumentFromReader(io.LimitReader(resp.Body, maxBodyBytes))
 	if err != nil {
 		return nil, err
 	}
@@ -256,7 +258,7 @@ func extractContentFromURL(url string) (string, error) {
 		return "", fmt.Errorf("HTTP %d", resp.StatusCode)
 	}
 
-	doc, err := goquery.NewDocumentFromReader(resp.Body)
+	doc, err := goquery.NewDocumentFromReader(io.LimitReader(resp.Body, maxBodyBytes))
 	if err != nil {
 		return "", err
 	}

backend/cmd/server/main.go

@@ -170,8 +170,11 @@ func main() {
 		}
 
 		api.GET("/search", handlers.SearchNews)
+		api.GET("/search/suggestions", middleware.AuthRequired(), handlers.SearchSuggestions)
+		api.POST("/searchlog", middleware.AuthRequired(), handlers.LogSearch)
 
 		api.GET("/entities", handlers.GetEntities)
+		api.GET("/entities/news", handlers.GetEntityNews)
 
 		api.GET("/stats", handlers.GetStats)
 
@@ -193,6 +196,7 @@ func main() {
 			admin.POST("/users/:id/demote", handlers.DemoteUser)
 			admin.POST("/reset-db", handlers.ResetDatabase)
 			admin.GET("/workers/status", handlers.GetWorkerStatus)
+			admin.GET("/workers/stats", handlers.GetTranslationStats)
 			admin.POST("/workers/config", handlers.SetWorkerConfig)
 			admin.POST("/workers/start", handlers.StartWorkers)
 			admin.POST("/workers/stop", handlers.StopWorkers)

backend/cmd/topics/main.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"context"
+	"fmt"
 	"log"
 	"os"
 	"os/signal"
@@ -276,29 +277,46 @@ func processBatch(ctx context.Context, topics []Topic, countries []Country) (int
 		processedIDs = append(processedIDs, item.ID)
 	}
 
-	// Insert topic relations
+	// Insert topic relations (batched, upsert)
 	if len(topicMatches) > 0 {
-		for _, tm := range topicMatches {
-			_, err := dbPool.Exec(ctx, `
+		const chunkSize = 500
+		for i := 0; i < len(topicMatches); i += chunkSize {
+			end := i + chunkSize
+			if end > len(topicMatches) {
+				end = len(topicMatches)
+			}
+			chunk := topicMatches[i:end]
+			values := make([]string, 0, len(chunk))
+			args := make([]interface{}, 0, len(chunk)*3)
+			for j, tm := range chunk {
+				values = append(values, fmt.Sprintf("($%d, $%d, $%d)", j*3+1, j*3+2, j*3+3))
+				args = append(args, tm.NoticiaID, tm.TopicID, tm.Score)
+			}
+			query := fmt.Sprintf(`
 				INSERT INTO news_topics (noticia_id, topic_id, score)
-				VALUES ($1, $2, $3)
+				VALUES %s
 				ON CONFLICT (noticia_id, topic_id) DO UPDATE SET score = EXCLUDED.score
-			`, tm.NoticiaID, tm.TopicID, tm.Score)
-			if err != nil {
-				logger.Printf("Error inserting topic: %v", err)
+			`, strings.Join(values, ","))
+			if _, err := dbPool.Exec(ctx, query, args...); err != nil {
+				logger.Printf("Error batch inserting topics: %v", err)
 			}
 		}
 	}
 
-	// Update country
+	// Update country (single bulk UPDATE)
 	if len(countryUpdates) > 0 {
-		for _, cu := range countryUpdates {
-			_, err := dbPool.Exec(ctx, `
-				UPDATE noticias SET pais_id = $1 WHERE id = $2
-			`, cu.PaisID, cu.NoticiaID)
-			if err != nil {
-				logger.Printf("Error updating country: %v", err)
-			}
+		paisSlice := make([]int32, len(countryUpdates))
+		idSlice := make([]string, len(countryUpdates))
+		for i, cu := range countryUpdates {
+			paisSlice[i] = int32(cu.PaisID)
+			idSlice[i] = cu.NoticiaID
+		}
+		if _, err := dbPool.Exec(ctx, `
+			UPDATE noticias n SET pais_id = u.pais
+			FROM unnest($1::int[], $2::varchar[]) AS u(pais, noticia_id)
+			WHERE n.id = u.noticia_id
+		`, paisSlice, idSlice); err != nil {
+			logger.Printf("Error bulk updating countries: %v", err)
 		}
 	}
 

backend/cmd/wiki_worker/main.go

@@ -23,8 +23,9 @@ var (
 	logger        *log.Logger
 	pool          *pgxpool.Pool
 	sleepInterval = 30
-	batchSize     = 50
+	batchSize     = 20
 	imagesDir     = "/app/data/wiki_images"
+	maxImageBytes = int64(2 << 20)
 )
 
 type WikiSummary struct {
@@ -101,13 +102,17 @@ func downloadImage(imgURL, destPath string) error {
 		return fmt.Errorf("HTTP %d", resp.StatusCode)
 	}
 
+	if resp.ContentLength > maxImageBytes {
+		return fmt.Errorf("image too large (%d bytes)", resp.ContentLength)
+	}
+
 	out, err := os.Create(destPath)
 	if err != nil {
 		return err
 	}
 	defer out.Close()
 
-	_, err = io.Copy(out, resp.Body)
+	_, err = io.Copy(out, io.LimitReader(resp.Body, maxImageBytes))
 	return err
 }
 
@@ -210,9 +215,15 @@ func processTag(ctx context.Context, tag Tag) {
 }
 
 func main() {
+	var sleepVal, batchVal int
 	if val := os.Getenv("WIKI_SLEEP"); val != "" {
-		if sleep, err := fmt.Sscanf(val, "%d", &sleepInterval); err == nil && sleep > 0 {
-			sleepInterval = sleep
+		if _, err := fmt.Sscanf(val, "%d", &sleepVal); err == nil && sleepVal > 0 {
+			sleepInterval = sleepVal
+		}
+	}
+	if val := os.Getenv("WIKI_BATCH"); val != "" {
+		if _, err := fmt.Sscanf(val, "%d", &batchVal); err == nil && batchVal > 0 {
+			batchSize = batchVal
 		}
 	}