pietre/rss2
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

scrapper

Browse source
This commit is contained in:
lordpietre 2026-03-31 22:57:47 +00:00
parent ee90335b92
commit 313a7c8b19
4 changed files with 158 additions and 13 deletions
Download patch file Download diff file Expand all files Collapse all files

3
Dockerfile.scraper
View file

@ -27,6 +27,7 @@ ENV DB_HOST=db \
DB_USER=rss \
DB_PASS=rss \
SCRAPER_SLEEP=60 \
SCRAPER_BATCH=10
SCRAPER_BATCH=10 \
SCRAPER_ENRICH_LIMIT=20
ENTRYPOINT ["/bin/scraper"]

163
backend/cmd/scraper/main.go
View file

@ -24,6 +24,7 @@ var (
pool *pgxpool.Pool
sleepInterval = 60
batchSize = 10
enrichLimit = 20
)
type URLSource struct {
@ -36,6 +37,14 @@ type URLSource struct {
Active bool
}
type Noticia struct {
ID string
Titulo string
Resumen string
URL string
FuenteNombre string
}
type Article struct {
Title string
Summary string
@ -53,6 +62,7 @@ func init() {
func loadConfig() {
sleepInterval = getEnvInt("SCRAPER_SLEEP", 60)
batchSize = getEnvInt("SCRAPER_BATCH", 10)
enrichLimit = getEnvInt("SCRAPER_ENRICH_LIMIT", 20)
}
func getEnvInt(key string, defaultValue int) int {
@ -66,9 +76,9 @@ func getEnvInt(key string, defaultValue int) int {
func getActiveURLs(ctx context.Context) ([]URLSource, error) {
rows, err := pool.Query(ctx, `
SELECT id, nombre, url, categoria_id, pais_id, idioma, activo
FROM fuentes_url
WHERE activo = true
SELECT id, nombre, url, categoria_id, pais_id, idioma, active
FROM fuentes_url
WHERE active = true
`)
if err != nil {
return nil, err
@ -89,16 +99,49 @@ func getActiveURLs(ctx context.Context) ([]URLSource, error) {
func updateSourceStatus(ctx context.Context, sourceID int64, status, message string, httpCode int) error {
_, err := pool.Exec(ctx, `
UPDATE fuentes_url
SET last_check = NOW(),
last_status = $1,
status_message = $2,
last_http_code = $3
WHERE id = $4
UPDATE fuentes_url
SET last_check = NOW(),
last_status = $1,
status_message = $2,
last_http_code = $3
WHERE id = $4
`, status, message, httpCode, sourceID)
return err
}
func getNoticiasToEnrich(ctx context.Context, limit int) ([]Noticia, error) {
rows, err := pool.Query(ctx, `
SELECT id, titulo, COALESCE(resumen, ''), url, fuente_nombre
FROM noticias
WHERE (resumen IS NULL OR LENGTH(resumen) < 200)
ORDER BY fecha DESC
LIMIT $1
`, limit)
if err != nil {
return nil, err
}
defer rows.Close()
var noticias []Noticia
for rows.Next() {
var n Noticia
if err := rows.Scan(&n.ID, &n.Titulo, &n.Resumen, &n.URL, &n.FuenteNombre); err != nil {
continue
}
noticias = append(noticias, n)
}
return noticias, nil
}
func updateNoticiaResumen(ctx context.Context, noticiaID, newResumen string) error {
_, err := pool.Exec(ctx, `
UPDATE noticias
SET resumen = $1
WHERE id = $2
`, newResumen, noticiaID)
return err
}
func extractArticle(source URLSource) (*Article, error) {
client := &http.Client{
Timeout: 30 * time.Second,
@ -186,6 +229,94 @@ func extractArticle(source URLSource) (*Article, error) {
return article, nil
}
func extractContentFromURL(url string) (string, error) {
client := &http.Client{
Timeout: 30 * time.Second,
}
req, err := http.NewRequest("GET", url, nil)
if err != nil {
return "", err
}
req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36")
req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")
req.Header.Set("Accept-Language", "en-US,en;q=0.5")
resp, err := client.Do(req)
if err != nil {
return "", err
}
defer resp.Body.Close()
if resp.StatusCode != 200 {
return "", fmt.Errorf("HTTP %d", resp.StatusCode)
}
doc, err := goquery.NewDocumentFromReader(resp.Body)
if err != nil {
return "", err
}
contentSelectors := []string{
"article",
"[role='main']",
"main",
".article-content",
".post-content",
".entry-content",
".content",
"#content",
".story-body",
".article-body",
}
var content string
for _, sel := range contentSelectors {
content = doc.Find(sel).First().Text()
if len(content) > 100 {
break
}
}
if len(content) < 100 {
ps := doc.Find("p")
var paragraphs []string
ps.Each(func(i int, s *goquery.Selection) {
txt := strings.TrimSpace(s.Text())
if len(txt) > 50 {
paragraphs = append(paragraphs, txt)
}
})
content = strings.Join(paragraphs, "\n\n")
}
return strings.TrimSpace(content), nil
}
func processEnrichment(ctx context.Context, noticia Noticia) bool {
logger.Printf("Enriching: %s", noticia.Titulo)
content, err := extractContentFromURL(noticia.URL)
if err != nil {
logger.Printf("Error extracting content from %s: %v", noticia.URL, err)
return false
}
if len(content) < 50 {
logger.Printf("Not enough content extracted from %s", noticia.URL)
return false
}
if err := updateNoticiaResumen(ctx, noticia.ID, content); err != nil {
logger.Printf("Error updating resumen for %s: %v", noticia.ID, err)
return false
}
logger.Printf("Enriched: %s (content length: %d)", noticia.Titulo, len(content))
return true
}
func saveArticle(ctx context.Context, source URLSource, article *Article) (bool, error) {
finalURL := article.URL
if finalURL == "" {
@ -300,7 +431,7 @@ func main() {
os.Exit(0)
}()
logger.Printf("Config: sleep=%ds, batch=%d", sleepInterval, batchSize)
logger.Printf("Config: sleep=%ds, batch=%d, enrich=%d", sleepInterval, batchSize, enrichLimit)
ticker := time.NewTicker(time.Duration(sleepInterval) * time.Second)
defer ticker.Stop()
@ -308,6 +439,18 @@ func main() {
for {
select {
case <-ticker.C:
noticias, err := getNoticiasToEnrich(ctx, enrichLimit)
if err != nil {
logger.Printf("Error fetching noticias to enrich: %v", err)
} else if len(noticias) > 0 {
logger.Printf("Enriching %d noticias", len(noticias))
for _, noticia := range noticias {
if processEnrichment(ctx, noticia) {
time.Sleep(3 * time.Second)
}
}
}
sources, err := getActiveURLs(ctx)
if err != nil {
logger.Printf("Error fetching URLs: %v", err)

3
docker-compose.yml
View file

@ -139,6 +139,7 @@ services:
DB_PASS: ${DB_PASS}
SCRAPER_SLEEP: 60
SCRAPER_BATCH: 10
SCRAPER_ENRICH_LIMIT: 20
TZ: Europe/Madrid
networks:
- backend
@ -262,7 +263,7 @@ services:
image: nginx:alpine
container_name: rss2_nginx
ports:
- "8001:80"
- "8888:80"
volumes:
- ./nginx.conf:/etc/nginx/nginx.conf:ro
networks:

2
frontend/vite.config.ts
View file

@ -7,7 +7,7 @@ export default defineConfig({
port: 3000,
proxy: {
'/api': {
target: 'http://localhost:8080',
target: 'http://localhost:8888',
changeOrigin: true,
},
},