83 lines
3.2 KiB
Python
83 lines
3.2 KiB
Python
import hashlib
|
|
from datetime import datetime
|
|
import logging
|
|
import newspaper
|
|
from newspaper import Config
|
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
|
|
def _process_individual_article(article_url, config):
|
|
"""
|
|
Función auxiliar que descarga y procesa un solo artículo.
|
|
Está diseñada para ser ejecutada en un hilo separado.
|
|
"""
|
|
try:
|
|
article = newspaper.Article(article_url, config=config)
|
|
article.download()
|
|
article.parse()
|
|
|
|
if not article.title or not article.text:
|
|
return None
|
|
|
|
article.nlp()
|
|
return article
|
|
except Exception:
|
|
return None
|
|
|
|
def process_newspaper_url(url, categoria_id, pais_id, idioma='es'):
|
|
"""
|
|
Explora la URL de un periódico, extrae los artículos que encuentra
|
|
en paralelo y devuelve una lista de noticias listas para la base de datos.
|
|
"""
|
|
logging.info(f"Iniciando el scrapeo en paralelo de la fuente: {url} (idioma: {idioma})")
|
|
|
|
todas_las_noticias = []
|
|
|
|
try:
|
|
config = Config()
|
|
config.browser_user_agent = 'RssApp/1.0 (Scraper)'
|
|
config.request_timeout = 15
|
|
config.memoize_articles = False
|
|
|
|
# Usamos el idioma proporcionado para mejorar la extracción
|
|
source = newspaper.build(url, config=config, language=idioma)
|
|
|
|
articles_to_process = source.articles[:25]
|
|
|
|
logging.info(f"Fuente construida. Procesando {len(articles_to_process)} artículos en paralelo...")
|
|
|
|
with ThreadPoolExecutor(max_workers=10) as executor:
|
|
future_to_article = {executor.submit(_process_individual_article, article.url, config): article for article in articles_to_process}
|
|
|
|
for future in as_completed(future_to_article):
|
|
processed_article = future.result()
|
|
|
|
if processed_article:
|
|
noticia_id = hashlib.md5(processed_article.url.encode()).hexdigest()
|
|
|
|
if processed_article.summary:
|
|
resumen = processed_article.summary
|
|
else:
|
|
resumen = (processed_article.text[:400] + '...') if len(processed_article.text) > 400 else processed_article.text
|
|
|
|
fecha = processed_article.publish_date if processed_article.publish_date else datetime.now()
|
|
|
|
todas_las_noticias.append((
|
|
noticia_id,
|
|
processed_article.title,
|
|
resumen,
|
|
processed_article.url,
|
|
fecha,
|
|
processed_article.top_image or '',
|
|
categoria_id,
|
|
pais_id
|
|
))
|
|
|
|
if not todas_las_noticias:
|
|
return [], "No se encontraron artículos válidos en la URL proporcionada."
|
|
|
|
return todas_las_noticias, f"Se procesaron {len(todas_las_noticias)} noticias con éxito."
|
|
|
|
except Exception as e:
|
|
logging.error(f"Excepción al construir la fuente desde '{url}': {e}", exc_info=True)
|
|
return [], f"Error al explorar la URL principal: {e}"
|
|
|