pietre/rss2
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

cambios

Browse source
This commit is contained in:
jlimolina 2026-08-28 21:10:02 +02:00
parent 3991237685
commit a01050bf85
46 changed files with 3636 additions and 652 deletions
Download patch file Download diff file Expand all files Collapse all files

45
backend/cmd/discovery/main.go
View file

@ -14,11 +14,11 @@ import (
"github.com/jackc/pgx/v5/pgxpool"
"github.com/mmcdole/gofeed"
"github.com/rss2/backend/internal/logger"
"github.com/rss2/backend/internal/workers"
)
var (
logger *log.Logger
pool *workers.Config
dbPool *pgxpool.Pool
sleepSec = 900 // 15 minutes
@ -35,7 +35,8 @@ type URLSource struct {
}
func init() {
logger = log.New(os.Stdout, "[DISCOVERY] ", log.LstdFlags)
logLevel := os.Getenv("LOG_LEVEL")
logger.Init("discovery-worker", logLevel)
}
func loadConfig() {
@ -416,48 +417,70 @@ func processURLSource(ctx context.Context, source URLSource) {
func main() {
loadConfig()
logger.Println("Starting RSS Discovery Worker")
logger.Info().Msg("Starting RSS Discovery Worker")
cfg := workers.LoadDBConfig()
if err := workers.Connect(cfg); err != nil {
logger.Fatalf("Failed to connect to database: %v", err)
logger.Fatal().Err(err).Msg("Failed to connect to database")
}
dbPool = workers.GetPool()
defer workers.Close()
logger.Println("Connected to PostgreSQL")
logger.Info().Msg("Connected to PostgreSQL")
ctx := context.Background()
// Start health check HTTP server
go func() {
http.HandleFunc("/health", func(w http.ResponseWriter, r *http.Request) {
if err := workers.HealthCheck(r.Context()); err != nil {
http.Error(w, "unhealthy", http.StatusServiceUnavailable)
return
}
w.WriteHeader(http.StatusOK)
w.Write([]byte("ok"))
})
logger.Info().Msg("Health check server listening on :8080")
if err := http.ListenAndServe(":8080", nil); err != nil {
logger.Error().Err(err).Msg("Health check server error")
}
}()
ctx, cancel := context.WithCancel(context.Background())
sigChan := make(chan os.Signal, 1)
signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM)
go func() {
<-sigChan
logger.Println("Shutting down...")
logger.Info().Msg("Shutting down gracefully...")
cancel()
time.Sleep(2 * time.Second)
workers.Close()
os.Exit(0)
}()
logger.Printf("Config: interval=%ds, batch=%d", sleepSec, batchSize)
logger.Info().Msgf("Config: interval=%ds, batch=%d", sleepSec, batchSize)
ticker := time.NewTicker(time.Duration(sleepSec) * time.Second)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
logger.Info().Msg("Context cancelled, stopping worker")
return
case <-ticker.C:
sources, err := getPendingURLs(ctx)
if err != nil {
logger.Printf("Error fetching URLs: %v", err)
logger.Error().Err(err).Msg("Error fetching URLs")
continue
}
if len(sources) == 0 {
logger.Println("No pending URLs to process")
logger.Info().Msg("No pending URLs to process")
continue
}
logger.Printf("Processing %d sources", len(sources))
logger.Info().Msgf("Processing %d sources", len(sources))
for _, source := range sources {
processURLSource(ctx, source)

70
backend/cmd/qdrant/main.go
View file

@ -6,7 +6,6 @@ import (
"encoding/json"
"fmt"
"io"
"log"
"net/http"
"os"
"os/signal"
@ -17,11 +16,11 @@ import (
"github.com/google/uuid"
"github.com/jackc/pgx/v5/pgxpool"
"github.com/rss2/backend/internal/logger"
"github.com/rss2/backend/internal/workers"
)
var (
logger *log.Logger
dbPool *pgxpool.Pool
qdrantURL string
collection = "news_vectors"
@ -30,28 +29,29 @@ var (
)
func init() {
logger = log.New(os.Stdout, "[QDRANT] ", log.LstdFlags)
logLevel := os.Getenv("LOG_LEVEL")
logger.Init("qdrant-worker", logLevel)
}
func loadConfig() {
sleepSec = getEnvInt("QDRANT_SLEEP", 30)
batchSize = getEnvInt("QDRANT_BATCH", 100)
qdrantHost := getEnv("QDRANT_HOST", "localhost")
qdrantPort := getEnvInt("QDRANT_PORT", 6333)
qdrantURL = fmt.Sprintf("http://%s:%d", qdrantHost, qdrantPort)
collection = getEnv("QDRANT_COLLECTION", "news_vectors")
batchSize = getEnvInt("QDRANT_BATCH",100)
qdrantHost := getEnv("QDRANT_HOST","localhost")
qdrantPort := getEnvInt("QDRANT_PORT",6333)
qdrantURL = fmt.Sprintf("http://%s:%d",qdrantHost,qdrantPort)
collection = getEnv("QDRANT_COLLECTION","news_vectors")
}
func getEnv(key, defaultValue string) string {
func getEnv(key,defaultValue string) string {
if value := os.Getenv(key); value != "" {
return value
}
return defaultValue
}
func getEnvInt(key string, defaultValue int) int {
func getEnvInt(key string,defaultValue int) int {
if value := os.Getenv(key); value != "" {
if intVal, err := strconv.Atoi(value); err == nil {
if intVal,err := strconv.Atoi(value); err == nil {
return intVal
}
}
@ -310,17 +310,33 @@ func main() {
cfg := workers.LoadDBConfig()
if err := workers.Connect(cfg); err != nil {
logger.Fatalf("Failed to connect to database: %v", err)
logger.Fatal().Err(err).Msg("Failed to connect to database")
}
dbPool = workers.GetPool()
defer workers.Close()
logger.Println("Connected to PostgreSQL")
logger.Info().Msg("Connected to PostgreSQL")
ctx := context.Background()
// Start health check HTTP server
go func() {
http.HandleFunc("/health", func(w http.ResponseWriter, r *http.Request) {
if err := workers.HealthCheck(r.Context()); err != nil {
http.Error(w, "unhealthy", http.StatusServiceUnavailable)
return
}
w.WriteHeader(http.StatusOK)
w.Write([]byte("ok"))
})
logger.Info().Msg("Health check server listening on :8080")
if err := http.ListenAndServe(":8080", nil); err != nil {
logger.Error().Err(err).Msg("Health check server error")
}
}()
ctx, cancel := context.WithCancel(context.Background())
if err := ensureCollection(ctx); err != nil {
logger.Printf("Warning: Could not ensure collection: %v", err)
logger.Warn().Err(err).Msg("Could not ensure collection")
}
sigChan := make(chan os.Signal, 1)
@ -328,30 +344,36 @@ func main() {
go func() {
<-sigChan
logger.Println("Shutting down...")
logger.Info().Msg("Shutting down gracefully...")
cancel()
time.Sleep(2 * time.Second)
workers.Close()
os.Exit(0)
}()
logger.Printf("Config: qdrant=%s, collection=%s, sleep=%ds, batch=%d",
logger.Info().Msgf("Config: qdrant=%s, collection=%s, sleep=%ds, batch=%d",
qdrantURL, collection, sleepSec, batchSize)
totalProcessed := 0
for {
select {
case <-ctx.Done():
logger.Info().Msg("Context cancelled, stopping worker")
return
case <-time.After(time.Duration(sleepSec) * time.Second):
translations, err := getPendingTranslations(ctx)
if err != nil {
logger.Printf("Error fetching pending translations: %v", err)
logger.Error().Err(err).Msg("Error fetching pending translations")
continue
}
if len(translations) == 0 {
logger.Println("No pending translations to process")
logger.Info().Msg("No pending translations to process")
continue
}
logger.Printf("Processing %d translations...", len(translations))
logger.Info().Msgf("Processing %d translations...", len(translations))
// Generate point IDs once (used both in Qdrant and DB)
pointIDs := make([]string, len(translations))
@ -361,21 +383,21 @@ func main() {
// Upload to Qdrant
if err := uploadToQdrant(translations, pointIDs); err != nil {
logger.Printf("Error uploading to Qdrant: %v", err)
logger.Error().Err(err).Msg("Error uploading to Qdrant")
continue
}
// Update DB status
if err := updateTranslationStatus(ctx, translations, pointIDs); err != nil {
logger.Printf("Error updating status: %v", err)
logger.Error().Err(err).Msg("Error updating status")
}
totalProcessed += len(translations)
logger.Printf("Processed %d translations (total: %d)", len(translations), totalProcessed)
logger.Info().Msgf("Processed %d translations (total: %d)", len(translations), totalProcessed)
total, vectorized, pending, err := getStats(ctx)
if err == nil {
logger.Printf("Stats: total=%d, vectorized=%d, pending=%d", total, vectorized, pending)
logger.Info().Msgf("Stats: total=%d, vectorized=%d, pending=%d", total, vectorized, pending)
}
}
}

44
backend/cmd/related/main.go
View file

@ -3,6 +3,7 @@ package main
import (
"context"
"log"
"net/http"
"os"
"os/signal"
"strconv"
@ -10,11 +11,11 @@ import (
"time"
"github.com/jackc/pgx/v5/pgxpool"
"github.com/rss2/backend/internal/logger"
"github.com/rss2/backend/internal/workers"
)
var (
logger *log.Logger
dbPool *pgxpool.Pool
sleepSec = 10
topK = 10
@ -23,7 +24,8 @@ var (
)
func init() {
logger = log.New(os.Stdout, "[RELATED] ", log.LstdFlags)
logLevel := os.Getenv("LOG_LEVEL")
logger.Init("related-worker", logLevel)
}
func loadConfig() {
@ -335,20 +337,36 @@ func processBatch(ctx context.Context, model string) (int, error) {
func main() {
loadConfig()
logger.Println("Starting Related News Worker")
logger.Info().Msg("Starting Related News Worker")
cfg := workers.LoadDBConfig()
if err := workers.Connect(cfg); err != nil {
logger.Fatalf("Failed to connect to database: %v", err)
logger.Fatal().Err(err).Msg("Failed to connect to database")
}
dbPool = workers.GetPool()
defer workers.Close()
ctx := context.Background()
// Start health check HTTP server
go func() {
http.HandleFunc("/health", func(w http.ResponseWriter, r *http.Request) {
if err := workers.HealthCheck(r.Context()); err != nil {
http.Error(w, "unhealthy", http.StatusServiceUnavailable)
return
}
w.WriteHeader(http.StatusOK)
w.Write([]byte("ok"))
})
logger.Info().Msg("Health check server listening on :8080")
if err := http.ListenAndServe(":8080", nil); err != nil {
logger.Error().Err(err).Msg("Health check server error")
}
}()
ctx, cancel := context.WithCancel(context.Background())
// Ensure schema
if err := ensureSchema(ctx); err != nil {
logger.Printf("Error ensuring schema: %v", err)
logger.Error().Err(err).Msg("Error ensuring schema")
}
model := os.Getenv("EMB_MODEL")
@ -361,23 +379,29 @@ func main() {
go func() {
<-sigChan
logger.Println("Shutting down...")
logger.Info().Msg("Shutting down gracefully...")
cancel()
time.Sleep(2 * time.Second)
workers.Close()
os.Exit(0)
}()
logger.Printf("Config: sleep=%ds, topK=%d, batch=%d, model=%s", sleepSec, topK, batchSz, model)
logger.Info().Msgf("Config: sleep=%ds, topK=%d, batch=%d, model=%s", sleepSec, topK, batchSz, model)
for {
select {
case <-ctx.Done():
logger.Info().Msg("Context cancelled, stopping worker")
return
case <-time.After(time.Duration(sleepSec) * time.Second):
count, err := processBatch(ctx, model)
if err != nil {
logger.Printf("Error processing batch: %v", err)
logger.Error().Err(err).Msg("Error processing batch")
continue
}
if count > 0 {
logger.Printf("Generated related news for %d translations", count)
logger.Info().Msgf("Generated related news for %d translations", count)
}
}
}

76
backend/cmd/scraper/main.go
View file

@ -5,7 +5,6 @@ import (
"crypto/md5"
"fmt"
"io"
"log"
"net/http"
"os"
"os/signal"
@ -17,12 +16,11 @@ import (
"github.com/PuerkitoBio/goquery"
"github.com/jackc/pgx/v5/pgxpool"
"github.com/rss2/backend/internal/logger"
"github.com/rss2/backend/internal/workers"
)
var (
logger *log.Logger
dbPool *workers.Config
pool *pgxpool.Pool
sleepInterval = 60
batchSize = 10
@ -59,15 +57,31 @@ type Article struct {
}
func init() {
logger = log.New(os.Stdout, "[SCRAPER] ", log.LstdFlags)
logger.SetOutput(os.Stdout)
logLevel := os.Getenv("LOG_LEVEL")
logger.Init("scraper-worker", logLevel)
}
func loadConfig() {
sleepInterval = getEnvInt("SCRAPER_SLEEP", 60)
batchSize = getEnvInt("SCRAPER_BATCH", 10)
enrichLimit = getEnvInt("SCRAPER_ENRICH_LIMIT", 20)
scraperWorkers = getEnvInt("SCRAPER_WORKERS", 5)
if v := os.Getenv("SCRAPER_INTERVAL"); v != "" {
if i, err := strconv.Atoi(v); err == nil && i > 0 {
sleepInterval = i
}
}
if v := os.Getenv("SCRAPER_BATCH"); v != "" {
if i, err := strconv.Atoi(v); err == nil && i > 0 {
batchSize = i
}
}
if v := os.Getenv("SCRAPER_ENRICH_LIMIT"); v != "" {
if i, err := strconv.Atoi(v); err == nil && i > 0 {
enrichLimit = i
}
}
if v := os.Getenv("SCRAPER_WORKERS"); v != "" {
if i, err := strconv.Atoi(v); err == nil && i > 0 {
scraperWorkers = i
}
}
}
func getEnvInt(key string, defaultValue int) int {
@ -448,18 +462,34 @@ func scrapeSources(ctx context.Context, sources []URLSource) {
func main() {
loadConfig()
logger.Println("Starting Scraper Worker")
logger.Info().Msg("Starting Scraper Worker")
cfg := workers.LoadDBConfig()
if err := workers.Connect(cfg); err != nil {
logger.Fatalf("Failed to connect to database: %v", err)
logger.Fatal().Err(err).Msg("Failed to connect to database")
}
pool = workers.GetPool()
defer workers.Close()
logger.Println("Connected to PostgreSQL")
logger.Info().Msg("Connected to PostgreSQL")
ctx := context.Background()
// Start health check HTTP server
go func() {
http.HandleFunc("/health", func(w http.ResponseWriter, r *http.Request) {
if err := workers.HealthCheck(r.Context()); err != nil {
http.Error(w, "unhealthy", http.StatusServiceUnavailable)
return
}
w.WriteHeader(http.StatusOK)
w.Write([]byte("ok"))
})
logger.Info().Msg("Health check server listening on :8080")
if err := http.ListenAndServe(":8080", nil); err != nil {
logger.Error().Err(err).Msg("Health check server error")
}
}()
ctx, cancel := context.WithCancel(context.Background())
// Handle shutdown
sigChan := make(chan os.Signal, 1)
@ -467,38 +497,44 @@ func main() {
go func() {
<-sigChan
logger.Println("Shutting down...")
logger.Info().Msg("Shutting down gracefully...")
cancel()
time.Sleep(2 * time.Second)
workers.Close()
os.Exit(0)
}()
logger.Printf("Config: sleep=%ds, batch=%d, enrich=%d", sleepInterval, batchSize, enrichLimit)
logger.Info().Msgf("Config: sleep=%ds, batch=%d, enrich=%d", sleepInterval, batchSize, enrichLimit)
ticker := time.NewTicker(time.Duration(sleepInterval) * time.Second)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
logger.Info().Msg("Context cancelled, stopping worker")
return
case <-ticker.C:
noticias, err := getNoticiasToEnrich(ctx, enrichLimit)
if err != nil {
logger.Printf("Error fetching noticias to enrich: %v", err)
logger.Error().Err(err).Msg("Error fetching noticias to enrich")
} else if len(noticias) > 0 {
logger.Printf("Enriching %d noticias (%d workers)", len(noticias), scraperWorkers)
logger.Info().Msgf("Enriching %d noticias (%d workers)", len(noticias), scraperWorkers)
enrichConcurrent(ctx, noticias)
}
sources, err := getActiveURLs(ctx)
if err != nil {
logger.Printf("Error fetching URLs: %v", err)
logger.Error().Err(err).Msg("Error fetching URLs")
continue
}
if len(sources) == 0 {
logger.Println("No active URLs to process")
logger.Info().Msg("No active URLs to process")
continue
}
logger.Printf("Processing %d sources (%d workers)", len(sources), scraperWorkers)
logger.Info().Msgf("Processing %d sources (%d workers)", len(sources), scraperWorkers)
scrapeSources(ctx, sources)
}
}

80
backend/cmd/server/main.go
View file

@ -3,7 +3,6 @@ package main
import (
"context"
"fmt"
"log"
"os"
"os/signal"
"syscall"
@ -14,12 +13,17 @@ import (
"github.com/rss2/backend/internal/config"
"github.com/rss2/backend/internal/db"
"github.com/rss2/backend/internal/handlers"
"github.com/rss2/backend/internal/logger"
"github.com/rss2/backend/internal/middleware"
"github.com/rss2/backend/internal/services"
swaggerFiles "github.com/swaggo/files"
ginSwagger "github.com/swaggo/gin-swagger"
_ "github.com/rss2/backend/docs"
)
func initDB() {
ctx := context.Background()
log := logger.GetLogger()
// Crear tabla entity_aliases si no existe
_, err := db.GetPool().Exec(ctx, `
@ -33,9 +37,9 @@ func initDB() {
)
`)
if err != nil {
log.Printf("Warning: Could not create entity_aliases table: %v", err)
log.Warn().Err(err).Msg("Could not create entity_aliases table")
} else {
log.Println("Table entity_aliases ready")
log.Info().Msg("Table entity_aliases ready")
}
// Añadir columna role a users si no existe
@ -43,9 +47,9 @@ func initDB() {
ALTER TABLE users ADD COLUMN IF NOT EXISTS role VARCHAR(20) DEFAULT 'user'
`)
if err != nil {
log.Printf("Warning: Could not add role column: %v", err)
log.Warn().Err(err).Msg("Could not add role column")
} else {
log.Println("Column role ready")
log.Info().Msg("Column role ready")
}
// Crear tabla de configuración si no existe
@ -129,31 +133,66 @@ func initDB() {
if err != nil {
log.Printf("Warning: Could not add assigned_at column: %v", err)
}
// Crear índices para optimizar consultas críticas
indexes := []string{
"CREATE INDEX IF NOT EXISTS idx_noticias_fecha ON noticias(fecha DESC)",
"CREATE INDEX IF NOT EXISTS idx_noticias_fuente_nombre ON noticias(fuente_nombre)",
"CREATE INDEX IF NOT EXISTS idx_noticias_categoria_id ON noticias(categoria_id)",
"CREATE INDEX IF NOT EXISTS idx_noticias_pais_id ON noticias(pais_id)",
"CREATE INDEX IF NOT EXISTS idx_noticias_lang ON noticias(lang)",
"CREATE INDEX IF NOT EXISTS idx_traducciones_noticia_id_lang_to ON traducciones(noticia_id, lang_to)",
"CREATE INDEX IF NOT EXISTS idx_traducciones_status ON traducciones(status)",
"CREATE INDEX IF NOT EXISTS idx_tags_noticia_noticia_id_tag_id ON tags_noticia(noticia_id, tag_id)",
"CREATE INDEX IF NOT EXISTS idx_tags_noticia_traduccion_id ON tags_noticia(traduccion_id)",
"CREATE INDEX IF NOT EXISTS idx_tags_valor_tipo ON tags(valor, tipo)",
"CREATE INDEX IF NOT EXISTS idx_entity_aliases_alias_tipo ON entity_aliases(alias, tipo)",
"CREATE INDEX IF NOT EXISTS idx_alertas_valor_tipo_periodo ON alertas(valor, tipo, periodo)",
"CREATE INDEX IF NOT EXISTS idx_alertas_status ON alertas(status)",
"CREATE INDEX IF NOT EXISTS idx_user_search_tags_user_id ON user_search_tags(user_id)",
"CREATE INDEX IF NOT EXISTS idx_feeds_activo ON feeds(activo)",
"CREATE INDEX IF NOT EXISTS idx_feeds_categoria_id ON feeds(categoria_id)",
"CREATE INDEX IF NOT EXISTS idx_feeds_pais_id ON feeds(pais_id)",
}
for _, idx := range indexes {
_, err := db.GetPool().Exec(ctx, idx)
if err != nil {
log.Warn().Err(err).Msg("Could not create index")
} else {
log.Info().Msg("Index created/verified")
}
}
}
func main() {
cfg := config.Load()
// Initialize structured logger
logLevel := os.Getenv("LOG_LEVEL")
logger.Init("rss2-api", logLevel)
log := logger.GetLogger()
if err := db.Connect(cfg.DatabaseURL); err != nil {
log.Fatalf("Failed to connect to database: %v", err)
log.Fatal().Err(err).Msg("Failed to connect to database")
}
defer db.Close()
log.Println("Connected to PostgreSQL")
log.Info().Msg("Connected to PostgreSQL")
// Auto-setup DB tables
initDB()
if err := cache.Connect(cfg.RedisURL); err != nil {
log.Printf("Warning: Failed to connect to Redis: %v", err)
log.Warn().Err(err).Msg("Failed to connect to Redis")
} else {
defer cache.Close()
log.Println("Connected to Redis")
log.Info().Msg("Connected to Redis")
}
services.Init(cfg)
r := gin.Default()
r.Use(middleware.RequestIDMiddleware())
r.Use(middleware.CORSMiddleware())
r.Use(middleware.LoggerMiddleware())
@ -161,14 +200,25 @@ func main() {
c.JSON(200, gin.H{"status": "ok"})
})
// Swagger documentation
r.GET("/swagger/*any", ginSwagger.WrapHandler(swaggerFiles.Handler))
api := r.Group("/api")
{
// Serve static images downloaded by wiki_worker
api.StaticFS("/wiki-images", gin.Dir(cfg.WikiImagesPath, false))
api.POST("/auth/login", handlers.Login)
api.POST("/auth/register", handlers.Register)
api.GET("/auth/check-first-user", handlers.CheckFirstUser)
// Stricter rate limiting for auth endpoints
authGroup := api.Group("/auth")
authGroup.Use(middleware.RateLimitMiddleware(10)) // 10 req/min for auth
{
authGroup.POST("/login", handlers.Login)
authGroup.POST("/register", handlers.Register)
authGroup.GET("/check-first-user", handlers.CheckFirstUser)
}
// General rate limiting for API
api.Use(middleware.RateLimitMiddleware(cfg.RateLimitPerMinute))
news := api.Group("/news")
{
@ -251,9 +301,9 @@ func main() {
addr := fmt.Sprintf(":%s", port)
go func() {
log.Printf("Server starting on %s", addr)
log.Info().Str("addr", addr).Msg("Server starting")
if err := r.Run(addr); err != nil {
log.Fatalf("Failed to start server: %v", err)
log.Fatal().Err(err).Msg("Failed to start server")
}
}()
@ -264,5 +314,5 @@ func main() {
signal.Notify(quit, syscall.SIGINT, syscall.SIGTERM)
<-quit
log.Println("Shutting down server...")
log.Info().Msg("Shutting down server...")
}

50
backend/cmd/topics/main.go
View file

@ -4,6 +4,7 @@ import (
"context"
"fmt"
"log"
"net/http"
"os"
"os/signal"
"strconv"
@ -12,11 +13,11 @@ import (
"time"
"github.com/jackc/pgx/v5/pgxpool"
"github.com/rss2/backend/internal/logger"
"github.com/rss2/backend/internal/workers"
)
var (
logger *log.Logger
dbPool *pgxpool.Pool
sleepSec = 10
batchSz = 500
@ -35,7 +36,8 @@ type Country struct {
}
func init() {
logger = log.New(os.Stdout, "[TOPICS] ", log.LstdFlags)
logLevel := os.Getenv("LOG_LEVEL")
logger.Init("topics-worker", logLevel)
}
func loadConfig() {
@ -335,20 +337,36 @@ func processBatch(ctx context.Context, topics []Topic, countries []Country) (int
func main() {
loadConfig()
logger.Println("Starting Topics Worker")
logger.Info().Msg("Starting Topics Worker")
cfg := workers.LoadDBConfig()
if err := workers.Connect(cfg); err != nil {
logger.Fatalf("Failed to connect to database: %v", err)
logger.Fatal().Err(err).Msg("Failed to connect to database")
}
dbPool = workers.GetPool()
defer workers.Close()
ctx := context.Background()
// Start health check HTTP server
go func() {
http.HandleFunc("/health", func(w http.ResponseWriter, r *http.Request) {
if err := workers.HealthCheck(r.Context()); err != nil {
http.Error(w, "unhealthy", http.StatusServiceUnavailable)
return
}
w.WriteHeader(http.StatusOK)
w.Write([]byte("ok"))
})
logger.Info().Msg("Health check server listening on :8080")
if err := http.ListenAndServe(":8080", nil); err != nil {
logger.Error().Err(err).Msg("Health check server error")
}
}()
ctx, cancel := context.WithCancel(context.Background())
// Ensure schema
if err := ensureSchema(ctx); err != nil {
logger.Printf("Error ensuring schema: %v", err)
logger.Error().Err(err).Msg("Error ensuring schema")
}
sigChan := make(chan os.Signal, 1)
@ -356,41 +374,47 @@ func main() {
go func() {
<-sigChan
logger.Println("Shutting down...")
logger.Info().Msg("Shutting down gracefully...")
cancel()
time.Sleep(2 * time.Second)
workers.Close()
os.Exit(0)
}()
logger.Printf("Config: sleep=%ds, batch=%d", sleepSec, batchSz)
logger.Info().Msgf("Config: sleep=%ds, batch=%d", sleepSec, batchSz)
for {
select {
case <-ctx.Done():
logger.Info().Msg("Context cancelled, stopping worker")
return
case <-time.After(time.Duration(sleepSec) * time.Second):
topics, err := loadTopics(ctx)
if err != nil {
logger.Printf("Error loading topics: %v", err)
logger.Error().Err(err).Msg("Error loading topics")
continue
}
if len(topics) == 0 {
logger.Println("No topics found in DB")
logger.Info().Msg("No topics found in DB")
time.Sleep(time.Duration(sleepSec) * time.Second)
continue
}
countries, err := loadCountries(ctx)
if err != nil {
logger.Printf("Error loading countries: %v", err)
logger.Error().Err(err).Msg("Error loading countries")
continue
}
count, err := processBatch(ctx, topics, countries)
if err != nil {
logger.Printf("Error processing batch: %v", err)
logger.Error().Err(err).Msg("Error processing batch")
continue
}
if count > 0 {
logger.Printf("Processed %d news items", count)
logger.Info().Msgf("Processed %d news items", count)
}
if count < batchSz {

84
backend/cmd/wiki_worker/main.go
View file

@ -16,11 +16,11 @@ import (
"time"
"github.com/jackc/pgx/v5/pgxpool"
"github.com/rss2/backend/internal/logger"
"github.com/rss2/backend/internal/workers"
)
var (
logger *log.Logger
pool *pgxpool.Pool
sleepInterval = 30
batchSize = 20
@ -52,7 +52,8 @@ type Tag struct {
}
func init() {
logger = log.New(os.Stdout, "[WIKI_WORKER] ", log.LstdFlags)
logLevel := os.Getenv("LOG_LEVEL")
logger.Init("wiki-worker", logLevel)
}
func getPendingTags(ctx context.Context) ([]Tag, error) {
@ -227,52 +228,89 @@ func main() {
}
}
logger.Println("Iniciando Wiki Worker...")
logger.Info().Msg("Iniciando Wiki Worker...")
if err := os.MkdirAll(imagesDir, 0755); err != nil {
logger.Fatalf("Error creando directorio de imágenes: %v", err)
logger.Fatal().Err(err).Msg("Error creando directorio de imágenes")
}
cfg := workers.LoadDBConfig()
if err := workers.Connect(cfg); err != nil {
logger.Fatalf("Failed to connect to database: %v", err)
logger.Fatal().Err(err).Msg("Failed to connect to database")
}
pool = workers.GetPool()
defer workers.Close()
ctx := context.Background()
// Start health check HTTP server
go func() {
http.HandleFunc("/health", func(w http.ResponseWriter, r *http.Request) {
if err := workers.HealthCheck(r.Context()); err != nil {
http.Error(w, "unhealthy", http.StatusServiceUnavailable)
return
}
w.WriteHeader(http.StatusOK)
w.Write([]byte("ok"))
})
logger.Info().Msg("Health check server listening on :8080")
if err := http.ListenAndServe(":8080", nil); err != nil {
logger.Error().Err(err).Msg("Health check server error")
}
}()
ctx, cancel := context.WithCancel(context.Background())
sigChan := make(chan os.Signal, 1)
signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM)
go func() {
<-sigChan
logger.Println("Cerrando gracefully...")
logger.Info().Msg("Shutting down gracefully...")
cancel()
// Give time for in-flight requests to complete
time.Sleep(2 * time.Second)
workers.Close()
os.Exit(0)
}()
logger.Printf("Configuración: sleep=%ds, batch=%d", sleepInterval, batchSize)
logger.Info().Msgf("Configuración: sleep=%ds, batch=%d", sleepInterval, batchSize)
for {
tags, err := getPendingTags(ctx)
if err != nil {
logger.Printf("Error recuperando tags pendientes: %v", err)
time.Sleep(10 * time.Second)
continue
}
select {
case <-ctx.Done():
logger.Info().Msg("Context cancelled, stopping worker")
return
default:
tags, err := getPendingTags(ctx)
if err != nil {
logger.Error().Err(err).Msg("Error recuperando tags pendientes")
select {
case <-ctx.Done():
return
case <-time.After(10 * time.Second):
}
continue
}
if len(tags) == 0 {
logger.Printf("No hay tags pendientes. Durmiendo %d segundos...", sleepInterval)
time.Sleep(time.Duration(sleepInterval) * time.Second)
continue
}
if len(tags) == 0 {
logger.Info().Msgf("No hay tags pendientes. Durmiendo %d segundos...", sleepInterval)
select {
case <-ctx.Done():
return
case <-time.After(time.Duration(sleepInterval) * time.Second):
}
continue
}
logger.Printf("Recuperados %d tags para procesar...", len(tags))
logger.Info().Msgf("Recuperados %d tags para procesar...", len(tags))
for _, tag := range tags {
processTag(ctx, tag)
time.Sleep(3 * time.Second) // Increased delay to avoid Wikipedia Rate Limits (429)
for _, tag := range tags {
processTag(ctx, tag)
select {
case <-ctx.Done():
return
case <-time.After(3 * time.Second): // Increased delay to avoid Wikipedia Rate Limits (429)
}
}
}
}
}