feat: add remote worker system for distributed translation
- Add remote translator worker (Docker) that connects via WebSocket - Add backend Go handlers for remote worker CRUD (REST + WebSocket) - Add frontend Admin panel for managing remote workers - Add ProtectedRoute component to secure admin routes - Add PostgreSQL schema for remote_workers table - Fix duplicate Load function in config.go - Fix frontend duplicate fetchStatus function - Fix Dockerfiles missing go.sum (use go mod download + tidy) Features: - Workers connect to server via WebSocket (no ports needed) - API key authentication per worker - Job assigner automatically distributes translation jobs - Worker status tracking (online/offline/disabled) - Timeout handling for stuck jobs
This commit is contained in:
parent
b8e48f3199
commit
8093cc1e40
21 changed files with 2065 additions and 58 deletions
|
|
@ -4,10 +4,11 @@ WORKDIR /app
|
|||
|
||||
RUN apt-get update && apt-get install -y gcc musl-dev git
|
||||
|
||||
COPY go.mod go.sum ./
|
||||
COPY go.mod ./
|
||||
RUN go mod download
|
||||
|
||||
COPY . .
|
||||
RUN go mod tidy
|
||||
|
||||
RUN CGO_ENABLED=0 GOOS=linux go build -buildvcs=false -o /server ./cmd/server
|
||||
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ import (
|
|||
"syscall"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/rss2/backend/internal/auth"
|
||||
"github.com/rss2/backend/internal/cache"
|
||||
"github.com/rss2/backend/internal/config"
|
||||
"github.com/rss2/backend/internal/db"
|
||||
|
|
@ -74,6 +75,39 @@ func initDB() {
|
|||
INSERT INTO config (key, value) VALUES ('translator_status', 'stopped')
|
||||
ON CONFLICT (key) DO NOTHING
|
||||
`)
|
||||
|
||||
// Crear tabla de remote_workers si no existe
|
||||
_, err = db.GetPool().Exec(ctx, `
|
||||
CREATE TABLE IF NOT EXISTS remote_workers (
|
||||
id SERIAL PRIMARY KEY,
|
||||
name VARCHAR(255) NOT NULL,
|
||||
api_key VARCHAR(64) UNIQUE NOT NULL,
|
||||
capabilities VARCHAR(50) DEFAULT 'cpu',
|
||||
status VARCHAR(20) DEFAULT 'offline',
|
||||
last_seen TIMESTAMP,
|
||||
created_at TIMESTAMP DEFAULT NOW()
|
||||
)
|
||||
`)
|
||||
if err != nil {
|
||||
log.Printf("Warning: Could not create remote_workers table: %v", err)
|
||||
} else {
|
||||
log.Println("Table remote_workers ready")
|
||||
}
|
||||
|
||||
// Añadir columnas a traducciones para workers remotos
|
||||
_, err = db.GetPool().Exec(ctx, `
|
||||
ALTER TABLE traducciones ADD COLUMN IF NOT EXISTS worker_id INTEGER REFERENCES remote_workers(id)
|
||||
`)
|
||||
if err != nil {
|
||||
log.Printf("Warning: Could not add worker_id column: %v", err)
|
||||
}
|
||||
|
||||
_, err = db.GetPool().Exec(ctx, `
|
||||
ALTER TABLE traducciones ADD COLUMN IF NOT EXISTS assigned_at TIMESTAMP
|
||||
`)
|
||||
if err != nil {
|
||||
log.Printf("Warning: Could not add assigned_at column: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func main() {
|
||||
|
|
@ -109,7 +143,7 @@ func main() {
|
|||
api := r.Group("/api")
|
||||
{
|
||||
// Serve static images downloaded by wiki_worker
|
||||
api.StaticFS("/wiki-images", gin.Dir("/app/data/wiki_images", false))
|
||||
api.StaticFS("/wiki-images", gin.Dir(cfg.WikiImagesPath, false))
|
||||
|
||||
api.POST("/auth/login", handlers.Login)
|
||||
api.POST("/auth/register", handlers.Register)
|
||||
|
|
@ -161,8 +195,17 @@ func main() {
|
|||
admin.POST("/workers/config", handlers.SetWorkerConfig)
|
||||
admin.POST("/workers/start", handlers.StartWorkers)
|
||||
admin.POST("/workers/stop", handlers.StopWorkers)
|
||||
|
||||
admin.GET("/workers/remote", handlers.ListRemoteWorkers)
|
||||
admin.POST("/workers/remote", handlers.CreateRemoteWorker)
|
||||
admin.GET("/workers/remote/:id", handlers.GetRemoteWorker)
|
||||
admin.DELETE("/workers/remote/:id", handlers.DeleteRemoteWorker)
|
||||
admin.POST("/workers/remote/:id/toggle", handlers.ToggleRemoteWorker)
|
||||
admin.POST("/workers/remote/:id/regenerate-key", handlers.RegenerateAPIKey)
|
||||
}
|
||||
|
||||
r.GET("/ws/worker", handlers.HandleWorkerWS)
|
||||
|
||||
auth := api.Group("/auth")
|
||||
auth.Use(middleware.AuthRequired())
|
||||
{
|
||||
|
|
@ -170,7 +213,7 @@ func main() {
|
|||
}
|
||||
}
|
||||
|
||||
middleware.SetJWTSecret(cfg.SecretKey)
|
||||
auth.SetJWTSecret(cfg.SecretKey)
|
||||
|
||||
port := cfg.ServerPort
|
||||
addr := fmt.Sprintf(":%s", port)
|
||||
|
|
@ -182,6 +225,8 @@ func main() {
|
|||
}
|
||||
}()
|
||||
|
||||
handlers.StartJobAssigner()
|
||||
|
||||
quit := make(chan os.Signal, 1)
|
||||
signal.Notify(quit, syscall.SIGINT, syscall.SIGTERM)
|
||||
<-quit
|
||||
|
|
|
|||
|
|
@ -20,23 +20,34 @@ type Config struct {
|
|||
DefaultLang string
|
||||
NewsPerPage int
|
||||
RateLimitPerMinute int
|
||||
DockerComposeDir string
|
||||
WikiImagesPath string
|
||||
AllowedOrigins string
|
||||
}
|
||||
|
||||
func Load() *Config {
|
||||
secretKey := os.Getenv("SECRET_KEY")
|
||||
if secretKey == "" {
|
||||
secretKey = "change-this-secret-key"
|
||||
}
|
||||
|
||||
return &Config{
|
||||
ServerPort: getEnv("SERVER_PORT", "8080"),
|
||||
DatabaseURL: getEnv("DATABASE_URL", "postgres://rss:rss@localhost:5432/rss"),
|
||||
RedisURL: getEnv("REDIS_URL", "redis://localhost:6379"),
|
||||
QdrantHost: getEnv("QDRANT_HOST", "localhost"),
|
||||
QdrantPort: getEnvInt("QDRANT_PORT", 6333),
|
||||
SecretKey: getEnv("SECRET_KEY", "change-this-secret-key"),
|
||||
JWTExpiration: getEnvDuration("JWT_EXPIRATION", 24*time.Hour),
|
||||
TranslationURL: getEnv("TRANSLATION_URL", "http://libretranslate:7790"),
|
||||
OllamaURL: getEnv("OLLAMA_URL", "http://ollama:11434"),
|
||||
SpacyURL: getEnv("SPACY_URL", "http://spacy:8000"),
|
||||
DefaultLang: getEnv("DEFAULT_LANG", "es"),
|
||||
NewsPerPage: getEnvInt("NEWS_PER_PAGE", 30),
|
||||
RateLimitPerMinute: getEnvInt("RATE_LIMIT_PER_MINUTE", 60),
|
||||
ServerPort: getEnv("SERVER_PORT", "8080"),
|
||||
DatabaseURL: getEnv("DATABASE_URL", "postgres://rss:rss@localhost:5432/rss"),
|
||||
RedisURL: getEnv("REDIS_URL", "redis://localhost:6379"),
|
||||
QdrantHost: getEnv("QDRANT_HOST", "localhost"),
|
||||
QdrantPort: getEnvInt("QDRANT_PORT", 6333),
|
||||
SecretKey: secretKey,
|
||||
JWTExpiration: getEnvDuration("JWT_EXPIRATION", 24*time.Hour),
|
||||
TranslationURL: getEnv("TRANSLATION_URL", "http://libretranslate:7790"),
|
||||
OllamaURL: getEnv("OLLAMA_URL", "http://ollama:11434"),
|
||||
SpacyURL: getEnv("SPACY_URL", "http://spacy:8000"),
|
||||
DefaultLang: getEnv("DEFAULT_LANG", "es"),
|
||||
NewsPerPage: getEnvInt("NEWS_PER_PAGE", 30),
|
||||
RateLimitPerMinute: getEnvInt("RATE_LIMIT_PER_MINUTE", 60),
|
||||
DockerComposeDir: getEnv("DOCKER_COMPOSE_DIR", "/datos/rss2"),
|
||||
WikiImagesPath: getEnv("WIKI_IMAGES_PATH", "/app/data/wiki_images"),
|
||||
AllowedOrigins: getEnv("ALLOWED_ORIGINS", "*"),
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -63,4 +74,4 @@ func getEnvDuration(key string, defaultValue time.Duration) time.Duration {
|
|||
}
|
||||
}
|
||||
return defaultValue
|
||||
}
|
||||
}
|
||||
187
backend/internal/handlers/remote_worker.go
Normal file
187
backend/internal/handlers/remote_worker.go
Normal file
|
|
@ -0,0 +1,187 @@
|
|||
package handlers
|
||||
|
||||
import (
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"log"
|
||||
"net/http"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/rss2/backend/internal/db"
|
||||
"github.com/rss2/backend/internal/models"
|
||||
)
|
||||
|
||||
func CreateRemoteWorker(c *gin.Context) {
|
||||
var req struct {
|
||||
Name string `json:"name" binding:"required"`
|
||||
Capabilities string `json:"capabilities"`
|
||||
}
|
||||
|
||||
if err := c.ShouldBindJSON(&req); err != nil {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": "Invalid request", "message": err.Error()})
|
||||
return
|
||||
}
|
||||
|
||||
if req.Capabilities == "" {
|
||||
req.Capabilities = "cpu"
|
||||
}
|
||||
|
||||
apiKey := generateAPIKey()
|
||||
|
||||
ctx := c.Request.Context()
|
||||
var workerID int
|
||||
err := db.GetPool().QueryRow(ctx, `
|
||||
INSERT INTO remote_workers (name, api_key, capabilities, status, created_at)
|
||||
VALUES ($1, $2, $3, 'offline', NOW())
|
||||
RETURNING id
|
||||
`, req.Name, apiKey, req.Capabilities).Scan(&workerID)
|
||||
|
||||
if err != nil {
|
||||
log.Printf("Error creating remote worker: %v", err)
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to create worker"})
|
||||
return
|
||||
}
|
||||
|
||||
c.JSON(http.StatusCreated, gin.H{
|
||||
"id": workerID,
|
||||
"name": req.Name,
|
||||
"api_key": apiKey,
|
||||
"capabilities": req.Capabilities,
|
||||
"status": "offline",
|
||||
})
|
||||
}
|
||||
|
||||
func ListRemoteWorkers(c *gin.Context) {
|
||||
ctx := c.Request.Context()
|
||||
|
||||
rows, err := db.GetPool().Query(ctx, `
|
||||
SELECT id, name, api_key, capabilities, status, last_seen, created_at
|
||||
FROM remote_workers
|
||||
ORDER BY created_at DESC
|
||||
`)
|
||||
if err != nil {
|
||||
log.Printf("Error listing remote workers: %v", err)
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to list workers"})
|
||||
return
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var workers []models.RemoteWorker
|
||||
for rows.Next() {
|
||||
var w models.RemoteWorker
|
||||
if err := rows.Scan(&w.ID, &w.Name, &w.APIKey, &w.Capabilities, &w.Status, &w.LastSeen, &w.CreatedAt); err != nil {
|
||||
log.Printf("Error scanning worker: %v", err)
|
||||
continue
|
||||
}
|
||||
workers = append(workers, w)
|
||||
}
|
||||
|
||||
if workers == nil {
|
||||
workers = []models.RemoteWorker{}
|
||||
}
|
||||
|
||||
c.JSON(http.StatusOK, workers)
|
||||
}
|
||||
|
||||
func GetRemoteWorker(c *gin.Context) {
|
||||
id := c.Param("id")
|
||||
ctx := c.Request.Context()
|
||||
|
||||
var w models.RemoteWorker
|
||||
err := db.GetPool().QueryRow(ctx, `
|
||||
SELECT id, name, api_key, capabilities, status, last_seen, created_at
|
||||
FROM remote_workers
|
||||
WHERE id = $1
|
||||
`, id).Scan(&w.ID, &w.Name, &w.APIKey, &w.Capabilities, &w.Status, &w.LastSeen, &w.CreatedAt)
|
||||
|
||||
if err != nil {
|
||||
c.JSON(http.StatusNotFound, gin.H{"error": "Worker not found"})
|
||||
return
|
||||
}
|
||||
|
||||
var stats struct {
|
||||
JobsCompleted int `json:"jobs_completed"`
|
||||
JobsPending int `json:"jobs_pending"`
|
||||
}
|
||||
|
||||
db.GetPool().QueryRow(ctx, `
|
||||
SELECT COUNT(*) FROM traducciones WHERE worker_id = $1 AND status = 'done'
|
||||
`, id).Scan(&stats.JobsCompleted)
|
||||
|
||||
db.GetPool().QueryRow(ctx, `
|
||||
SELECT COUNT(*) FROM traducciones WHERE worker_id = $1 AND status = 'pending'
|
||||
`, id).Scan(&stats.JobsPending)
|
||||
|
||||
c.JSON(http.StatusOK, gin.H{
|
||||
"worker": w,
|
||||
"stats": stats,
|
||||
})
|
||||
}
|
||||
|
||||
func DeleteRemoteWorker(c *gin.Context) {
|
||||
id := c.Param("id")
|
||||
ctx := c.Request.Context()
|
||||
|
||||
result, err := db.GetPool().Exec(ctx, `
|
||||
DELETE FROM remote_workers WHERE id = $1
|
||||
`, id)
|
||||
|
||||
if err != nil {
|
||||
log.Printf("Error deleting remote worker: %v", err)
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to delete worker"})
|
||||
return
|
||||
}
|
||||
|
||||
if result.RowsAffected() == 0 {
|
||||
c.JSON(http.StatusNotFound, gin.H{"error": "Worker not found"})
|
||||
return
|
||||
}
|
||||
|
||||
c.JSON(http.StatusOK, gin.H{"message": "Worker deleted"})
|
||||
}
|
||||
|
||||
func ToggleRemoteWorker(c *gin.Context) {
|
||||
id := c.Param("id")
|
||||
ctx := c.Request.Context()
|
||||
|
||||
var currentStatus string
|
||||
err := db.GetPool().QueryRow(ctx, `SELECT status FROM remote_workers WHERE id = $1`, id).Scan(¤tStatus)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusNotFound, gin.H{"error": "Worker not found"})
|
||||
return
|
||||
}
|
||||
|
||||
newStatus := "disabled"
|
||||
if currentStatus == "disabled" {
|
||||
newStatus = "offline"
|
||||
}
|
||||
|
||||
_, err = db.GetPool().Exec(ctx, `UPDATE remote_workers SET status = $1 WHERE id = $2`, newStatus, id)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to update worker status"})
|
||||
return
|
||||
}
|
||||
|
||||
c.JSON(http.StatusOK, gin.H{"status": newStatus})
|
||||
}
|
||||
|
||||
func RegenerateAPIKey(c *gin.Context) {
|
||||
id := c.Param("id")
|
||||
ctx := c.Request.Context()
|
||||
|
||||
newAPIKey := generateAPIKey()
|
||||
|
||||
_, err := db.GetPool().Exec(ctx, `UPDATE remote_workers SET api_key = $1 WHERE id = $2`, newAPIKey, id)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to regenerate API key"})
|
||||
return
|
||||
}
|
||||
|
||||
c.JSON(http.StatusOK, gin.H{"api_key": newAPIKey})
|
||||
}
|
||||
|
||||
func generateAPIKey() string {
|
||||
bytes := make([]byte, 32)
|
||||
rand.Read(bytes)
|
||||
return hex.EncodeToString(bytes)
|
||||
}
|
||||
297
backend/internal/handlers/worker_ws.go
Normal file
297
backend/internal/handlers/worker_ws.go
Normal file
|
|
@ -0,0 +1,297 @@
|
|||
package handlers
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"log"
|
||||
"net/http"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/gorilla/websocket"
|
||||
"github.com/rss2/backend/internal/db"
|
||||
"github.com/rss2/backend/internal/models"
|
||||
)
|
||||
|
||||
var upgrader = websocket.Upgrader{
|
||||
CheckOrigin: func(r *http.Request) bool { return true },
|
||||
}
|
||||
|
||||
type WSWorker struct {
|
||||
conn *websocket.Conn
|
||||
workerID int
|
||||
capabilities string
|
||||
workerName string
|
||||
lastHeartbeat time.Time
|
||||
mu sync.Mutex
|
||||
}
|
||||
|
||||
var (
|
||||
workers = make(map[int]*WSWorker)
|
||||
workersByAPI = make(map[string]*WSWorker)
|
||||
workersMu sync.RWMutex
|
||||
)
|
||||
|
||||
func HandleWorkerWS(c *gin.Context) {
|
||||
apiKey := c.Query("api_key")
|
||||
if apiKey == "" {
|
||||
apiKey = c.GetHeader("X-API-Key")
|
||||
}
|
||||
|
||||
if apiKey == "" {
|
||||
c.JSON(http.StatusUnauthorized, gin.H{"error": "API key required"})
|
||||
return
|
||||
}
|
||||
|
||||
ctx := c.Request.Context()
|
||||
|
||||
var workerID int
|
||||
var capabilities string
|
||||
err := db.GetPool().QueryRow(ctx, `
|
||||
SELECT id, capabilities FROM remote_workers
|
||||
WHERE api_key = $1 AND status != 'disabled'
|
||||
`, apiKey).Scan(&workerID, &capabilities)
|
||||
|
||||
if err != nil {
|
||||
c.JSON(http.StatusUnauthorized, gin.H{"error": "Invalid API key"})
|
||||
return
|
||||
}
|
||||
|
||||
conn, err := upgrader.Upgrade(c.Writer, c.Request, nil)
|
||||
if err != nil {
|
||||
log.Printf("WebSocket upgrade error: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
wsWorker := &WSWorker{
|
||||
conn: conn,
|
||||
workerID: workerID,
|
||||
capabilities: capabilities,
|
||||
lastHeartbeat: time.Now(),
|
||||
}
|
||||
|
||||
workersMu.Lock()
|
||||
workers[workerID] = wsWorker
|
||||
workersByAPI[apiKey] = wsWorker
|
||||
workersMu.Unlock()
|
||||
|
||||
db.GetPool().Exec(ctx, `
|
||||
UPDATE remote_workers SET status = 'online', last_seen = NOW() WHERE id = $1
|
||||
`, workerID)
|
||||
|
||||
go heartbeatLoop(wsWorker)
|
||||
go writeLoop(wsWorker)
|
||||
readLoop(wsWorker, apiKey)
|
||||
}
|
||||
|
||||
func readLoop(wsWorker *WSWorker, apiKey string) {
|
||||
defer func() {
|
||||
cleanupWorker(wsWorker, apiKey)
|
||||
}()
|
||||
|
||||
for {
|
||||
var msg models.WSClientMessage
|
||||
err := wsWorker.conn.ReadJSON(&msg)
|
||||
if err != nil {
|
||||
log.Printf("Worker %d read error: %v", wsWorker.workerID, err)
|
||||
break
|
||||
}
|
||||
|
||||
switch msg.Type {
|
||||
case "register":
|
||||
wsWorker.mu.Lock()
|
||||
if msg.WorkerName != "" {
|
||||
wsWorker.workerName = msg.WorkerName
|
||||
}
|
||||
if msg.Capabilities != "" {
|
||||
wsWorker.capabilities = msg.Capabilities
|
||||
}
|
||||
wsWorker.mu.Unlock()
|
||||
|
||||
sendWS(wsWorker, models.WSServerMessage{Type: "ack", Job: nil})
|
||||
|
||||
case "heartbeat":
|
||||
wsWorker.mu.Lock()
|
||||
wsWorker.lastHeartbeat = time.Now()
|
||||
wsWorker.mu.Unlock()
|
||||
sendWS(wsWorker, models.WSServerMessage{Type: "ack", Job: nil})
|
||||
|
||||
case "result":
|
||||
if msg.Result != nil {
|
||||
handleTranslationResult(wsWorker.workerID, msg.Result)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func writeLoop(wsWorker *WSWorker) {
|
||||
ticker := time.NewTicker(30 * time.Second)
|
||||
defer ticker.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ticker.C:
|
||||
if err := wsWorker.conn.WriteMessage(websocket.PingMessage, nil); err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func heartbeatLoop(wsWorker *WSWorker) {
|
||||
ticker := time.NewTicker(10 * time.Second)
|
||||
defer ticker.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ticker.C:
|
||||
wsWorker.mu.Lock()
|
||||
elapsed := time.Since(wsWorker.lastHeartbeat)
|
||||
wsWorker.mu.Unlock()
|
||||
|
||||
if elapsed > 60*time.Second {
|
||||
log.Printf("Worker %d heartbeat timeout", wsWorker.workerID)
|
||||
wsWorker.conn.Close()
|
||||
return
|
||||
}
|
||||
|
||||
sendWS(wsWorker, models.WSServerMessage{Type: "ping", Job: nil})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func sendWS(wsWorker *WSWorker, msg models.WSServerMessage) {
|
||||
data, _ := json.Marshal(msg)
|
||||
wsWorker.conn.WriteMessage(websocket.TextMessage, data)
|
||||
}
|
||||
|
||||
func cleanupWorker(wsWorker *WSWorker, apiKey string) {
|
||||
workersMu.Lock()
|
||||
delete(workers, wsWorker.workerID)
|
||||
delete(workersByAPI, apiKey)
|
||||
workersMu.Unlock()
|
||||
|
||||
ctx := context.Background()
|
||||
db.GetPool().Exec(ctx, `
|
||||
UPDATE remote_workers SET status = 'offline', last_seen = NOW() WHERE id = $1
|
||||
`, wsWorker.workerID)
|
||||
|
||||
wsWorker.conn.Close()
|
||||
|
||||
db.GetPool().Exec(ctx, `
|
||||
UPDATE traducciones
|
||||
SET status = 'pending', worker_id = NULL, assigned_at = NULL
|
||||
WHERE worker_id = $1 AND status = 'assigned'
|
||||
`, wsWorker.workerID)
|
||||
|
||||
log.Printf("Worker %d disconnected", wsWorker.workerID)
|
||||
}
|
||||
|
||||
func handleTranslationResult(workerID int, result *models.TranslationResult) {
|
||||
ctx := context.Background()
|
||||
|
||||
if result.Error != "" {
|
||||
db.GetPool().Exec(ctx, `
|
||||
UPDATE traducciones
|
||||
SET status = 'error', worker_id = NULL, assigned_at = NULL
|
||||
WHERE id = $1
|
||||
`, result.JobID)
|
||||
log.Printf("Job %d failed: %s", result.JobID, result.Error)
|
||||
return
|
||||
}
|
||||
|
||||
_, err := db.GetPool().Exec(ctx, `
|
||||
UPDATE traducciones
|
||||
SET titulo_trad = $1, resumen_trad = $2, status = 'done',
|
||||
worker_id = $3, assigned_at = NULL
|
||||
WHERE id = $4
|
||||
`, result.TitleTr, result.SummaryTr, workerID, result.JobID)
|
||||
|
||||
if err != nil {
|
||||
log.Printf("Error updating translation result: %v", err)
|
||||
}
|
||||
|
||||
db.GetPool().Exec(ctx, `
|
||||
UPDATE remote_workers SET last_seen = NOW() WHERE id = $1
|
||||
`, workerID)
|
||||
|
||||
log.Printf("Job %d completed by worker %d", result.JobID, workerID)
|
||||
}
|
||||
|
||||
func AssignJobToWorker(workerID int) *models.TranslationJob {
|
||||
ctx := context.Background()
|
||||
|
||||
workersMu.RLock()
|
||||
wsWorker, exists := workers[workerID]
|
||||
workersMu.RUnlock()
|
||||
|
||||
if !exists {
|
||||
return nil
|
||||
}
|
||||
|
||||
wsWorker.mu.Lock()
|
||||
workerCapabilities := wsWorker.capabilities
|
||||
wsWorker.mu.Unlock()
|
||||
|
||||
_ = workerCapabilities
|
||||
|
||||
var job models.TranslationJob
|
||||
err := db.GetPool().QueryRow(ctx, `
|
||||
SELECT t.id, t.noticia_id, t.lang_from, t.lang_to, n.titulo, n.resumen
|
||||
FROM traducciones t
|
||||
JOIN noticias n ON n.id = t.noticia_id
|
||||
WHERE t.status = 'pending'
|
||||
AND (t.worker_id IS NULL OR t.status = 'assigned' AND t.assigned_at < NOW() - INTERVAL '5 minutes')
|
||||
AND t.lang_to = 'es'
|
||||
AND (t.titulo_trad IS NULL OR t.resumen_trad IS NULL)
|
||||
ORDER BY n.fecha DESC
|
||||
LIMIT 1
|
||||
FOR UPDATE SKIP LOCKED
|
||||
`).Scan(&job.ID, &job.NewsID, &job.LangFrom, &job.LangTo, &job.Title, &job.Summary)
|
||||
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
db.GetPool().Exec(ctx, `
|
||||
UPDATE traducciones
|
||||
SET status = 'assigned', worker_id = $1, assigned_at = NOW()
|
||||
WHERE id = $2
|
||||
`, workerID, job.ID)
|
||||
|
||||
sendWS(wsWorker, models.WSServerMessage{
|
||||
Type: "job",
|
||||
Job: &job,
|
||||
})
|
||||
|
||||
return &job
|
||||
}
|
||||
|
||||
func StartJobAssigner() {
|
||||
go func() {
|
||||
ticker := time.NewTicker(5 * time.Second)
|
||||
defer ticker.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ticker.C:
|
||||
workersMu.RLock()
|
||||
for workerID := range workers {
|
||||
workersMu.RUnlock()
|
||||
AssignJobToWorker(workerID)
|
||||
workersMu.RLock()
|
||||
}
|
||||
workersMu.RUnlock()
|
||||
|
||||
ctx := context.Background()
|
||||
db.GetPool().Exec(ctx, `
|
||||
UPDATE traducciones
|
||||
SET status = 'pending', worker_id = NULL, assigned_at = NULL
|
||||
WHERE status = 'assigned'
|
||||
AND assigned_at < NOW() - INTERVAL '10 minutes'
|
||||
`)
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
|
@ -5,23 +5,10 @@ import (
|
|||
"strings"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/golang-jwt/jwt/v5"
|
||||
"github.com/rss2/backend/internal/auth"
|
||||
"github.com/rss2/backend/internal/config"
|
||||
)
|
||||
|
||||
var jwtSecret []byte
|
||||
|
||||
func SetJWTSecret(secret string) {
|
||||
jwtSecret = []byte(secret)
|
||||
}
|
||||
|
||||
type Claims struct {
|
||||
UserID int64 `json:"user_id"`
|
||||
Email string `json:"email"`
|
||||
Username string `json:"username"`
|
||||
IsAdmin bool `json:"is_admin"`
|
||||
jwt.RegisteredClaims
|
||||
}
|
||||
|
||||
func AuthRequired() gin.HandlerFunc {
|
||||
return func(c *gin.Context) {
|
||||
authHeader := c.GetHeader("Authorization")
|
||||
|
|
@ -38,12 +25,8 @@ func AuthRequired() gin.HandlerFunc {
|
|||
return
|
||||
}
|
||||
|
||||
claims := &Claims{}
|
||||
token, err := jwt.ParseWithClaims(tokenString, claims, func(token *jwt.Token) (interface{}, error) {
|
||||
return jwtSecret, nil
|
||||
})
|
||||
|
||||
if err != nil || !token.Valid {
|
||||
claims, err := auth.ValidateToken(tokenString)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusUnauthorized, gin.H{"error": "Invalid token"})
|
||||
c.Abort()
|
||||
return
|
||||
|
|
@ -63,7 +46,7 @@ func AdminRequired() gin.HandlerFunc {
|
|||
return
|
||||
}
|
||||
|
||||
claims := userVal.(*Claims)
|
||||
claims := userVal.(*auth.Claims)
|
||||
if !claims.IsAdmin {
|
||||
c.JSON(http.StatusForbidden, gin.H{"error": "Admin access required"})
|
||||
c.Abort()
|
||||
|
|
@ -75,9 +58,30 @@ func AdminRequired() gin.HandlerFunc {
|
|||
}
|
||||
|
||||
func CORSMiddleware() gin.HandlerFunc {
|
||||
cfg := config.Load()
|
||||
allowedOrigins := strings.Split(cfg.AllowedOrigins, ",")
|
||||
|
||||
return func(c *gin.Context) {
|
||||
c.Writer.Header().Set("Access-Control-Allow-Origin", "*")
|
||||
c.Writer.Header().Set("Access-Control-Allow-Credentials", "true")
|
||||
origin := c.Request.Header.Get("Origin")
|
||||
|
||||
allowed := false
|
||||
for _, o := range allowedOrigins {
|
||||
o = strings.TrimSpace(o)
|
||||
if o == "*" || o == origin {
|
||||
allowed = true
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if allowed {
|
||||
if cfg.AllowedOrigins == "*" {
|
||||
c.Writer.Header().Set("Access-Control-Allow-Origin", "*")
|
||||
} else {
|
||||
c.Writer.Header().Set("Access-Control-Allow-Origin", origin)
|
||||
c.Writer.Header().Set("Access-Control-Allow-Credentials", "true")
|
||||
}
|
||||
}
|
||||
|
||||
c.Writer.Header().Set("Access-Control-Allow-Headers", "Content-Type, Content-Length, Accept-Encoding, X-CSRF-Token, Authorization, accept, origin, Cache-Control, X-Requested-With")
|
||||
c.Writer.Header().Set("Access-Control-Allow-Methods", "POST, OPTIONS, GET, PUT, DELETE, PATCH")
|
||||
|
||||
|
|
|
|||
41
backend/internal/models/remote_worker.go
Normal file
41
backend/internal/models/remote_worker.go
Normal file
|
|
@ -0,0 +1,41 @@
|
|||
package models
|
||||
|
||||
import "time"
|
||||
|
||||
type RemoteWorker struct {
|
||||
ID int `json:"id"`
|
||||
Name string `json:"name"`
|
||||
APIKey string `json:"api_key,omitempty"`
|
||||
Capabilities string `json:"capabilities"`
|
||||
Status string `json:"status"`
|
||||
LastSeen *time.Time `json:"last_seen,omitempty"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
}
|
||||
|
||||
type TranslationJob struct {
|
||||
ID int64 `json:"id"`
|
||||
NewsID int64 `json:"noticia_id"`
|
||||
LangFrom string `json:"lang_from"`
|
||||
LangTo string `json:"lang_to"`
|
||||
Title string `json:"title"`
|
||||
Summary string `json:"summary"`
|
||||
}
|
||||
|
||||
type TranslationResult struct {
|
||||
JobID int64 `json:"job_id"`
|
||||
TitleTr string `json:"title_trad"`
|
||||
SummaryTr string `json:"resumen_trad"`
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
type WSClientMessage struct {
|
||||
Type string `json:"type"`
|
||||
Capabilities string `json:"capabilities,omitempty"`
|
||||
WorkerName string `json:"worker_name,omitempty"`
|
||||
Result *TranslationResult `json:"result,omitempty"`
|
||||
}
|
||||
|
||||
type WSServerMessage struct {
|
||||
Type string `json:"type"`
|
||||
Job *TranslationJob `json:"job,omitempty"`
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue