scrapper

Dockerfile.scraper

@@ -27,6 +27,7 @@ ENV DB_HOST=db \
     DB_USER=rss \
     DB_PASS=rss \
     SCRAPER_SLEEP=60 \
-    SCRAPER_BATCH=10
+    SCRAPER_BATCH=10 \
+    SCRAPER_ENRICH_LIMIT=20
 
 ENTRYPOINT ["/bin/scraper"]

backend/cmd/scraper/main.go

@@ -24,6 +24,7 @@ var (
 	pool          *pgxpool.Pool
 	sleepInterval = 60
 	batchSize     = 10
+	enrichLimit   = 20
 )
 
 type URLSource struct {
@@ -36,6 +37,14 @@ type URLSource struct {
 	Active      bool
 }
 
+type Noticia struct {
+	ID           string
+	Titulo       string
+	Resumen      string
+	URL          string
+	FuenteNombre string
+}
+
 type Article struct {
 	Title    string
 	Summary  string
@@ -53,6 +62,7 @@ func init() {
 func loadConfig() {
 	sleepInterval = getEnvInt("SCRAPER_SLEEP", 60)
 	batchSize = getEnvInt("SCRAPER_BATCH", 10)
+	enrichLimit = getEnvInt("SCRAPER_ENRICH_LIMIT", 20)
 }
 
 func getEnvInt(key string, defaultValue int) int {
@@ -66,9 +76,9 @@ func getEnvInt(key string, defaultValue int) int {
 
 func getActiveURLs(ctx context.Context) ([]URLSource, error) {
 	rows, err := pool.Query(ctx, `
-		SELECT id, nombre, url, categoria_id, pais_id, idioma, activo
+	SELECT id, nombre, url, categoria_id, pais_id, idioma, active
-		FROM fuentes_url 
+	FROM fuentes_url 
-		WHERE activo = true
+	WHERE active = true
 	`)
 	if err != nil {
 		return nil, err
@@ -89,16 +99,49 @@ func getActiveURLs(ctx context.Context) ([]URLSource, error) {
 
 func updateSourceStatus(ctx context.Context, sourceID int64, status, message string, httpCode int) error {
 	_, err := pool.Exec(ctx, `
-		UPDATE fuentes_url 
+	UPDATE fuentes_url 
-		SET last_check = NOW(),
+	SET last_check = NOW(),
-		    last_status = $1,
+	    last_status = $1,
-		    status_message = $2,
+	    status_message = $2,
-		    last_http_code = $3
+	    last_http_code = $3
-		WHERE id = $4
+	WHERE id = $4
 	`, status, message, httpCode, sourceID)
 	return err
 }
 
+func getNoticiasToEnrich(ctx context.Context, limit int) ([]Noticia, error) {
+	rows, err := pool.Query(ctx, `
+	SELECT id, titulo, COALESCE(resumen, ''), url, fuente_nombre
+	FROM noticias
+	WHERE (resumen IS NULL OR LENGTH(resumen) < 200)
+	ORDER BY fecha DESC
+	LIMIT $1
+	`, limit)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+
+	var noticias []Noticia
+	for rows.Next() {
+		var n Noticia
+		if err := rows.Scan(&n.ID, &n.Titulo, &n.Resumen, &n.URL, &n.FuenteNombre); err != nil {
+			continue
+		}
+		noticias = append(noticias, n)
+	}
+	return noticias, nil
+}
+
+func updateNoticiaResumen(ctx context.Context, noticiaID, newResumen string) error {
+	_, err := pool.Exec(ctx, `
+	UPDATE noticias
+	SET resumen = $1
+	WHERE id = $2
+	`, newResumen, noticiaID)
+	return err
+}
+
 func extractArticle(source URLSource) (*Article, error) {
 	client := &http.Client{
 		Timeout: 30 * time.Second,
@@ -186,6 +229,94 @@ func extractArticle(source URLSource) (*Article, error) {
 	return article, nil
 }
 
+func extractContentFromURL(url string) (string, error) {
+	client := &http.Client{
+		Timeout: 30 * time.Second,
+	}
+
+	req, err := http.NewRequest("GET", url, nil)
+	if err != nil {
+		return "", err
+	}
+
+	req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36")
+	req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")
+	req.Header.Set("Accept-Language", "en-US,en;q=0.5")
+
+	resp, err := client.Do(req)
+	if err != nil {
+		return "", err
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != 200 {
+		return "", fmt.Errorf("HTTP %d", resp.StatusCode)
+	}
+
+	doc, err := goquery.NewDocumentFromReader(resp.Body)
+	if err != nil {
+		return "", err
+	}
+
+	contentSelectors := []string{
+		"article",
+		"[role='main']",
+		"main",
+		".article-content",
+		".post-content",
+		".entry-content",
+		".content",
+		"#content",
+		".story-body",
+		".article-body",
+	}
+
+	var content string
+	for _, sel := range contentSelectors {
+		content = doc.Find(sel).First().Text()
+		if len(content) > 100 {
+			break
+		}
+	}
+
+	if len(content) < 100 {
+		ps := doc.Find("p")
+		var paragraphs []string
+		ps.Each(func(i int, s *goquery.Selection) {
+			txt := strings.TrimSpace(s.Text())
+			if len(txt) > 50 {
+				paragraphs = append(paragraphs, txt)
+			}
+		})
+		content = strings.Join(paragraphs, "\n\n")
+	}
+
+	return strings.TrimSpace(content), nil
+}
+
+func processEnrichment(ctx context.Context, noticia Noticia) bool {
+	logger.Printf("Enriching: %s", noticia.Titulo)
+
+	content, err := extractContentFromURL(noticia.URL)
+	if err != nil {
+		logger.Printf("Error extracting content from %s: %v", noticia.URL, err)
+		return false
+	}
+
+	if len(content) < 50 {
+		logger.Printf("Not enough content extracted from %s", noticia.URL)
+		return false
+	}
+
+	if err := updateNoticiaResumen(ctx, noticia.ID, content); err != nil {
+		logger.Printf("Error updating resumen for %s: %v", noticia.ID, err)
+		return false
+	}
+
+	logger.Printf("Enriched: %s (content length: %d)", noticia.Titulo, len(content))
+	return true
+}
+
 func saveArticle(ctx context.Context, source URLSource, article *Article) (bool, error) {
 	finalURL := article.URL
 	if finalURL == "" {
@@ -300,7 +431,7 @@ func main() {
 		os.Exit(0)
 	}()
 
-	logger.Printf("Config: sleep=%ds, batch=%d", sleepInterval, batchSize)
+	logger.Printf("Config: sleep=%ds, batch=%d, enrich=%d", sleepInterval, batchSize, enrichLimit)
 
 	ticker := time.NewTicker(time.Duration(sleepInterval) * time.Second)
 	defer ticker.Stop()
@@ -308,6 +439,18 @@ func main() {
 	for {
 		select {
 		case <-ticker.C:
+			noticias, err := getNoticiasToEnrich(ctx, enrichLimit)
+			if err != nil {
+				logger.Printf("Error fetching noticias to enrich: %v", err)
+			} else if len(noticias) > 0 {
+				logger.Printf("Enriching %d noticias", len(noticias))
+				for _, noticia := range noticias {
+					if processEnrichment(ctx, noticia) {
+						time.Sleep(3 * time.Second)
+					}
+				}
+			}
+
 			sources, err := getActiveURLs(ctx)
 			if err != nil {
 				logger.Printf("Error fetching URLs: %v", err)

docker-compose.yml

@@ -139,6 +139,7 @@ services:
       DB_PASS: ${DB_PASS}
       SCRAPER_SLEEP: 60
       SCRAPER_BATCH: 10
+      SCRAPER_ENRICH_LIMIT: 20
       TZ: Europe/Madrid
     networks:
       - backend
@@ -262,7 +263,7 @@ services:
     image: nginx:alpine
     container_name: rss2_nginx
     ports:
-      - "8001:80"
+      - "8888:80"
     volumes:
       - ./nginx.conf:/etc/nginx/nginx.conf:ro
     networks:

frontend/vite.config.ts

@@ -7,7 +7,7 @@ export default defineConfig({
     port: 3000,
     proxy: {
       '/api': {
-        target: 'http://localhost:8080',
+        target: 'http://localhost:8888',
         changeOrigin: true,
       },
     },