import requests
from bs4 import BeautifulSoup
from elasticsearch import Elasticsearch

# Inicializar conexión a Elasticsearch con autenticación y usando https
es = Elasticsearch(
    hosts=[{'host': 'localhost', 'port': 9200, 'scheme': 'https'}],
    basic_auth=('jesucristo_garcia', 'JESUCRISTO_/garcia'),
    verify_certs=False  # Si no tienes un certificado SSL válido, desactiva la verificación
)

def scrape_wikipedia(url):
    response = requests.get(url)
    if response.status_code == 200:
        soup = BeautifulSoup(response.content, 'html.parser')
        title = soup.find('title').text
        paragraphs = soup.find_all('p')
        text = " ".join(paragraph.text for paragraph in paragraphs)
        return title, text
    else:
        print(f'Failed to retrieve {url}')
        return None, None

# URLs de las páginas de Wikipedia a scrapear
wikipedia_urls = [
    'https://en.wikipedia.org/wiki/OpenAI',
    # ... añade más URLs aquí
]

for url in wikipedia_urls:
    title, text = scrape_wikipedia(url)
    if title and text:
        document = {
            'title': title,
            'text': text,
            'source': 'Wikipedia',
            'url': url
        }
        try:
            # Indexar en Elasticsearch
            es.index(index='documents', document=document)
        except Exception as e:
            print(f"Error al indexar documento: {e}")

print('Scraping completado.')
