pietre/rss2
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

muchas cosas

Browse source
This commit is contained in:
jlimolina 2026-08-11 02:37:48 +02:00
parent e3936fd4cf
commit eff656adea
20 changed files with 697 additions and 51 deletions
Download patch file Download diff file Expand all files Collapse all files

6
backend/cmd/scraper/main.go
View file

@ -4,6 +4,7 @@ import (
"context"
"crypto/md5"
"fmt"
"io"
"log"
"net/http"
"os"
@ -27,6 +28,7 @@ var (
batchSize = 10
enrichLimit = 20
scraperWorkers = 5
maxBodyBytes = int64(512 << 10)
)
type URLSource struct {
@ -169,7 +171,7 @@ func extractArticle(source URLSource) (*Article, error) {
return nil, fmt.Errorf("HTTP %d", resp.StatusCode)
}
doc, err := goquery.NewDocumentFromReader(resp.Body)
doc, err := goquery.NewDocumentFromReader(io.LimitReader(resp.Body, maxBodyBytes))
if err != nil {
return nil, err
}
@ -256,7 +258,7 @@ func extractContentFromURL(url string) (string, error) {
return "", fmt.Errorf("HTTP %d", resp.StatusCode)
}
doc, err := goquery.NewDocumentFromReader(resp.Body)
doc, err := goquery.NewDocumentFromReader(io.LimitReader(resp.Body, maxBodyBytes))
if err != nil {
return "", err
}