audio_pipeline

workers/transcribe/internal/config/config.go

@@ -0,0 +1,78 @@
+package config
+
+import (
+	"os"
+	"strconv"
+	"time"
+)
+
+type Config struct {
+	RabbitURL       string
+	InputQueue      string
+	OutputExchange  string
+	AnalyseQueue    string
+	TaggingQueue    string
+	InputExchange   string
+	InputRoutingKey string
+	Prefetch        int
+
+	NexaraBaseURL string
+	NexaraAPIKey  string
+	NexaraModel   string
+	NexaraTimeout time.Duration
+
+	PromptsSource  string
+	PromptsFile    string
+	PromptsBaseURL string
+	PromptsAPIKey  string
+	PromptsSection int
+}
+
+func Load() Config {
+	return Config{
+		RabbitURL:       getEnv("RABBITMQ_URL", "amqp://guest:guest@localhost:5672/"),
+		InputQueue:      getEnv("INPUT_QUEUE", "transcribe.tasks"),
+		OutputExchange:  getEnv("OUTPUT_EXCHANGE", "transcription_done"),
+		AnalyseQueue:    getEnv("ANALYSE_QUEUE", "analyse"),
+		TaggingQueue:    getEnv("TAGGING_QUEUE", "tagging"),
+		InputExchange:   getEnv("RABBITMQ_EXCHANGE", "audio_pipeline"),
+		InputRoutingKey: getEnv("RABBITMQ_ROUTING_KEY", "audio.new"),
+		Prefetch:        getInt("PREFETCH", 1),
+
+		NexaraBaseURL: getEnv("NEXARA_BASE_URL", "https://api.nexara.ru"),
+		NexaraAPIKey:  os.Getenv("NEXARA_API_KEY"),
+		NexaraModel:   getEnv("NEXARA_MODEL", "whisper-1"),
+		NexaraTimeout: getDuration("NEXARA_TIMEOUT", 10*time.Minute),
+
+		PromptsSource:  getEnv("PROMPTS_SOURCE", "static"),
+		PromptsFile:    getEnv("PROMPTS_FILE", "/app/configs/prompts.json"),
+		PromptsBaseURL: os.Getenv("PROMPTS_BASE_URL"),
+		PromptsAPIKey:  os.Getenv("PROMPTS_API_KEY"),
+		PromptsSection: getInt("PROMPTS_SECTION", 1),
+	}
+}
+
+func getEnv(key, def string) string {
+	if v := os.Getenv(key); v != "" {
+		return v
+	}
+	return def
+}
+
+func getInt(key string, def int) int {
+	if v := os.Getenv(key); v != "" {
+		if i, err := strconv.Atoi(v); err == nil {
+			return i
+		}
+	}
+	return def
+}
+
+func getDuration(key string, def time.Duration) time.Duration {
+	if v := os.Getenv(key); v != "" {
+		if d, err := time.ParseDuration(v); err == nil {
+			return d
+		}
+	}
+	return def
+}

workers/transcribe/internal/consumer/consumer.go

@@ -0,0 +1,172 @@
+package consumer
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"log/slog"
+	"time"
+
+	amqp "github.com/rabbitmq/amqp091-go"
+
+	"github.com/yourorg/transcribe/internal/config"
+	"github.com/yourorg/transcribe/internal/models"
+	"github.com/yourorg/transcribe/internal/nexara"
+	"github.com/yourorg/transcribe/internal/prompts"
+)
+
+type Consumer struct {
+	cfg     config.Config
+	ch      *amqp.Channel
+	nexara  *nexara.Client
+	prompts *prompts.Loader
+}
+
+func New(cfg config.Config, ch *amqp.Channel) (*Consumer, error) {
+	if err := setupTopology(ch, cfg); err != nil {
+		return nil, err
+	}
+	return &Consumer{
+		cfg:     cfg,
+		ch:      ch,
+		nexara:  nexara.New(cfg.NexaraBaseURL, cfg.NexaraAPIKey, cfg.NexaraModel, cfg.NexaraTimeout),
+		prompts: prompts.New(cfg.PromptsSource, cfg.PromptsFile, cfg.PromptsBaseURL, cfg.PromptsAPIKey, cfg.PromptsSection),
+	}, nil
+}
+
+func setupTopology(ch *amqp.Channel, cfg config.Config) error {
+	if err := ch.ExchangeDeclare("dlx", "direct", true, false, false, false, nil); err != nil {
+		return fmt.Errorf("declare dlx: %w", err)
+	}
+	if err := ch.ExchangeDeclare(cfg.InputExchange, "direct", true, false, false, false, nil); err != nil {
+		return fmt.Errorf("declare input exchange: %w", err)
+	}
+	if err := ch.ExchangeDeclare(cfg.OutputExchange, "fanout", true, false, false, false, nil); err != nil {
+		return fmt.Errorf("declare output exchange: %w", err)
+	}
+
+	dlqArgs := amqp.Table{
+		"x-dead-letter-exchange":    "dlx",
+		"x-dead-letter-routing-key": cfg.InputQueue + ".failed",
+	}
+	if _, err := ch.QueueDeclare(cfg.InputQueue, true, false, false, false, dlqArgs); err != nil {
+		return fmt.Errorf("declare input queue: %w", err)
+	}
+	if _, err := ch.QueueDeclare(cfg.InputQueue+".failed", true, false, false, false, nil); err != nil {
+		return fmt.Errorf("declare dlq: %w", err)
+	}
+	if err := ch.QueueBind(cfg.InputQueue+".failed", cfg.InputQueue+".failed", "dlx", false, nil); err != nil {
+		return fmt.Errorf("bind dlq: %w", err)
+	}
+	if err := ch.QueueBind(cfg.InputQueue, cfg.InputRoutingKey, cfg.InputExchange, false, nil); err != nil {
+		return fmt.Errorf("bind input queue: %w", err)
+	}
+
+	for _, q := range []string{cfg.AnalyseQueue, cfg.TaggingQueue} {
+		if _, err := ch.QueueDeclare(q, true, false, false, false, nil); err != nil {
+			return fmt.Errorf("declare queue %s: %w", q, err)
+		}
+		if err := ch.QueueBind(q, "", cfg.OutputExchange, false, nil); err != nil {
+			return fmt.Errorf("bind queue %s: %w", q, err)
+		}
+	}
+
+	return ch.Qos(cfg.Prefetch, 0, false)
+}
+
+func (c *Consumer) Run(ctx context.Context) error {
+	if err := c.ch.Confirm(false); err != nil {
+		return fmt.Errorf("confirm mode: %w", err)
+	}
+
+	msgs, err := c.ch.Consume(c.cfg.InputQueue, "", false, false, false, false, nil)
+	if err != nil {
+		return err
+	}
+
+	slog.Info("transcribe worker started", "queue", c.cfg.InputQueue, "output_exchange", c.cfg.OutputExchange)
+
+	for {
+		select {
+		case <-ctx.Done():
+			return nil
+		case d, ok := <-msgs:
+			if !ok {
+				return fmt.Errorf("delivery channel closed")
+			}
+			c.handle(ctx, d)
+		}
+	}
+}
+
+func (c *Consumer) handle(ctx context.Context, d amqp.Delivery) {
+	var task models.AudioTask
+	if err := json.Unmarshal(d.Body, &task); err != nil {
+		slog.Warn("bad message", "delivery_tag", d.DeliveryTag, "error", err)
+		_ = d.Nack(false, false)
+		return
+	}
+
+	slog.Info("message received", "task_id", task.TaskID, "file_path", task.FilePath, "filename", task.Filename)
+
+	txCtx, cancel := context.WithTimeout(ctx, c.cfg.NexaraTimeout+30*time.Second)
+	defer cancel()
+
+	text, lang, segments, err := c.nexara.TranscribeFile(txCtx, task.FilePath)
+	if err != nil {
+		slog.Warn("transcription failed", "task_id", task.TaskID, "error", err)
+		_ = d.Nack(false, false)
+		return
+	}
+
+	promptList, err := c.prompts.Load(txCtx)
+	if err != nil {
+		slog.Warn("prompts load failed", "task_id", task.TaskID, "error", err)
+		_ = d.Nack(false, false)
+		return
+	}
+
+	result := models.TranscriptionResult{
+		TaskID:        task.TaskID,
+		Filename:      task.Filename,
+		FilePath:      task.FilePath,
+		Transcription: text,
+		Language:      lang,
+		Segments:      segments,
+		Prompts:       promptList,
+		TranscribedAt: time.Now().Unix(),
+	}
+
+	body, err := json.Marshal(result)
+	if err != nil {
+		slog.Warn("marshal failed", "task_id", task.TaskID, "error", err)
+		_ = d.Nack(false, false)
+		return
+	}
+
+	confirms := c.ch.NotifyPublish(make(chan amqp.Confirmation, 1))
+	if err := c.ch.PublishWithContext(txCtx, c.cfg.OutputExchange, "", false, false, amqp.Publishing{
+		ContentType:  "application/json",
+		Body:         body,
+		DeliveryMode: amqp.Persistent,
+	}); err != nil {
+		slog.Warn("publish failed, requeue", "task_id", task.TaskID, "error", err)
+		_ = d.Nack(false, true)
+		return
+	}
+	select {
+	case confirm := <-confirms:
+		if !confirm.Ack {
+			slog.Warn("publish not confirmed, requeue", "task_id", task.TaskID)
+			_ = d.Nack(false, true)
+			return
+		}
+	case <-txCtx.Done():
+		slog.Warn("publish timeout, requeue", "task_id", task.TaskID)
+		_ = d.Nack(false, true)
+		return
+	}
+
+	slog.Info("transcribed", "task_id", task.TaskID, "language", lang, "chars", len(text), "segments", len(segments), "prompts", len(promptList))
+	_ = d.Ack(false)
+}

workers/transcribe/internal/models/models.go

@@ -0,0 +1,34 @@
+package models
+
+type AudioTask struct {
+	TaskID    string `json:"task_id"`
+	FilePath  string `json:"file_path"`
+	Filename  string `json:"filename"`
+	Size      int64  `json:"size"`
+	CreatedAt int64  `json:"created_at"`
+}
+
+type Segment struct {
+	Start float64 `json:"start"`
+	End   float64 `json:"end"`
+	Text  string  `json:"text"`
+}
+
+type Prompt struct {
+	ID        int    `json:"id"`
+	IDSection int    `json:"id_section"`
+	Name      string `json:"name"`
+	Prompt    string `json:"prompt"`
+	DtCreate  string `json:"dt_create"`
+}
+
+type TranscriptionResult struct {
+	TaskID        string    `json:"task_id"`
+	Filename      string    `json:"filename"`
+	FilePath      string    `json:"file_path"`
+	Transcription string    `json:"transcription"`
+	Language      string    `json:"language"`
+	Segments      []Segment `json:"segments,omitempty"`
+	Prompts       []Prompt  `json:"prompts"`
+	TranscribedAt int64     `json:"transcribed_at"`
+}

workers/transcribe/internal/nexara/nexara.go

@@ -0,0 +1,117 @@
+package nexara
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"mime/multipart"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/yourorg/transcribe/internal/models"
+)
+
+type Client struct {
+	apiURL     string
+	apiKey     string
+	model      string
+	httpClient *http.Client
+}
+
+func New(baseURL, apiKey, model string, timeout time.Duration) *Client {
+	baseURL = strings.TrimRight(baseURL, "/")
+	return &Client{
+		apiURL: baseURL + "/api/v1/audio/transcriptions",
+		apiKey: apiKey,
+		model:  model,
+		httpClient: &http.Client{
+			Timeout: timeout,
+		},
+	}
+}
+
+func (c *Client) TranscribeFile(ctx context.Context, path string) (text, language string, segments []models.Segment, err error) {
+	f, err := os.Open(path)
+	if err != nil {
+		return "", "", nil, fmt.Errorf("open file: %w", err)
+	}
+	defer f.Close()
+
+	body := &bytes.Buffer{}
+	writer := multipart.NewWriter(body)
+	part, err := writer.CreateFormFile("file", filepath.Base(path))
+	if err != nil {
+		return "", "", nil, err
+	}
+	if _, err := io.Copy(part, f); err != nil {
+		return "", "", nil, err
+	}
+	if c.model != "" {
+		if err := writer.WriteField("model", c.model); err != nil {
+			return "", "", nil, err
+		}
+	}
+	if err := writer.WriteField("response_format", "json"); err != nil {
+		return "", "", nil, err
+	}
+	if err := writer.Close(); err != nil {
+		return "", "", nil, err
+	}
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.apiURL, body)
+	if err != nil {
+		return "", "", nil, err
+	}
+	req.Header.Set("Content-Type", writer.FormDataContentType())
+	req.Header.Set("Authorization", "Bearer "+c.apiKey)
+
+	resp, err := c.httpClient.Do(req)
+	if err != nil {
+		return "", "", nil, fmt.Errorf("request: %w", err)
+	}
+	defer resp.Body.Close()
+
+	respBody, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return "", "", nil, err
+	}
+	if resp.StatusCode != http.StatusOK {
+		return "", "", nil, fmt.Errorf("status %d: %s", resp.StatusCode, string(respBody))
+	}
+
+	var raw map[string]any
+	if err := json.Unmarshal(respBody, &raw); err != nil {
+		return "", "", nil, fmt.Errorf("parse: %w", err)
+	}
+	if t, ok := raw["text"].(string); ok {
+		text = t
+	}
+	if lang, ok := raw["language"].(string); ok {
+		language = lang
+	}
+	if segs, ok := raw["segments"].([]any); ok {
+		for _, s := range segs {
+			m, ok := s.(map[string]any)
+			if !ok {
+				continue
+			}
+			var seg models.Segment
+			if v, ok := m["start"].(float64); ok {
+				seg.Start = v
+			}
+			if v, ok := m["end"].(float64); ok {
+				seg.End = v
+			}
+			if v, ok := m["text"].(string); ok {
+				seg.Text = v
+			}
+			segments = append(segments, seg)
+		}
+	}
+	return text, language, segments, nil
+}

workers/transcribe/internal/prompts/prompts.go

@@ -0,0 +1,100 @@
+package prompts
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/yourorg/transcribe/internal/models"
+)
+
+type Loader struct {
+	source    string
+	filePath  string
+	baseURL   string
+	apiKey    string
+	sectionID int
+	client    *http.Client
+}
+
+func New(source, filePath, baseURL, apiKey string, sectionID int) *Loader {
+	return &Loader{
+		source:    source,
+		filePath:  filePath,
+		baseURL:   strings.TrimRight(baseURL, "/"),
+		apiKey:    apiKey,
+		sectionID: sectionID,
+		client:    &http.Client{Timeout: 30 * time.Second},
+	}
+}
+
+func (l *Loader) Load(ctx context.Context) ([]models.Prompt, error) {
+	switch strings.ToLower(l.source) {
+	case "http":
+		return l.loadHTTP(ctx)
+	default:
+		return l.loadStatic()
+	}
+}
+
+func (l *Loader) loadStatic() ([]models.Prompt, error) {
+	data, err := os.ReadFile(l.filePath)
+	if err != nil {
+		return nil, fmt.Errorf("read prompts file: %w", err)
+	}
+	var prompts []models.Prompt
+	if err := json.Unmarshal(data, &prompts); err != nil {
+		return nil, fmt.Errorf("parse prompts file: %w", err)
+	}
+	return filterSection(prompts, l.sectionID), nil
+}
+
+func (l *Loader) loadHTTP(ctx context.Context) ([]models.Prompt, error) {
+	if l.baseURL == "" {
+		return nil, fmt.Errorf("PROMPTS_BASE_URL is required for http source")
+	}
+	url := fmt.Sprintf("%s/metrics/?id_section=%s", l.baseURL, strconv.Itoa(l.sectionID))
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
+	if err != nil {
+		return nil, err
+	}
+	if l.apiKey != "" {
+		req.Header.Set("Authorization", "Bearer "+l.apiKey)
+	}
+	resp, err := l.client.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return nil, err
+	}
+	if resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("prompts api status %d: %s", resp.StatusCode, string(body))
+	}
+	var prompts []models.Prompt
+	if err := json.Unmarshal(body, &prompts); err != nil {
+		return nil, fmt.Errorf("parse prompts response: %w", err)
+	}
+	return filterSection(prompts, l.sectionID), nil
+}
+
+func filterSection(prompts []models.Prompt, sectionID int) []models.Prompt {
+	if sectionID <= 0 {
+		return prompts
+	}
+	out := make([]models.Prompt, 0, len(prompts))
+	for _, p := range prompts {
+		if p.IDSection == sectionID {
+			out = append(out, p)
+		}
+	}
+	return out
+}