audio_pipeline

watcher/internal/config/config.go

@@ -0,0 +1,60 @@
+package config
+
+import (
+	"os"
+	"strconv"
+	"time"
+)
+
+type Config struct {
+	StorageRoot   string
+	IncomingDir   string
+	ProcessingDir string
+	FailedDir     string
+	PollInterval  time.Duration
+	StableWindow  time.Duration
+	StableChecks  int
+	RabbitURL     string
+	Exchange      string
+	RoutingKey    string
+}
+
+func Load() Config {
+	return Config{
+		StorageRoot:   getEnv("STORAGE_ROOT", "/data/storage"),
+		IncomingDir:   getEnv("INCOMING_DIR", "incoming"),
+		ProcessingDir: getEnv("PROCESSING_DIR", "processing"),
+		FailedDir:     getEnv("FAILED_DIR", "failed"),
+		PollInterval:  getDuration("POLL_INTERVAL", 5*time.Second),
+		StableWindow:  getDuration("STABLE_WINDOW", 2*time.Second),
+		StableChecks:  getInt("STABLE_CHECKS", 3),
+		RabbitURL:     getEnv("RABBITMQ_URL", "amqp://guest:guest@localhost:5672/"),
+		Exchange:      getEnv("RABBITMQ_EXCHANGE", "audio_pipeline"),
+		RoutingKey:    getEnv("RABBITMQ_ROUTING_KEY", "audio.new"),
+	}
+}
+
+func getEnv(key, def string) string {
+	if v := os.Getenv(key); v != "" {
+		return v
+	}
+	return def
+}
+
+func getInt(key string, def int) int {
+	if v := os.Getenv(key); v != "" {
+		if i, err := strconv.Atoi(v); err == nil {
+			return i
+		}
+	}
+	return def
+}
+
+func getDuration(key string, def time.Duration) time.Duration {
+	if v := os.Getenv(key); v != "" {
+		if d, err := time.ParseDuration(v); err == nil {
+			return d
+		}
+	}
+	return def
+}

watcher/internal/publisher/publisher.go

@@ -0,0 +1,58 @@
+package publisher
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"time"
+
+	amqp "github.com/rabbitmq/amqp091-go"
+)
+
+type AudioTask struct {
+	TaskID    string `json:"task_id"`
+	FilePath  string `json:"file_path"`
+	Filename  string `json:"filename"`
+	Size      int64  `json:"size"`
+	CreatedAt int64  `json:"created_at"`
+}
+
+type Publisher struct {
+	ch         *amqp.Channel
+	exchange   string
+	routingKey string
+}
+
+func New(ch *amqp.Channel, exchange, routingKey string) (*Publisher, error) {
+	if err := ch.Confirm(false); err != nil {
+		return nil, fmt.Errorf("confirm mode: %w", err)
+	}
+	return &Publisher{ch: ch, exchange: exchange, routingKey: routingKey}, nil
+}
+
+func (p *Publisher) Publish(ctx context.Context, task AudioTask) error {
+	if task.CreatedAt == 0 {
+		task.CreatedAt = time.Now().Unix()
+	}
+	body, err := json.Marshal(task)
+	if err != nil {
+		return err
+	}
+	confirms := p.ch.NotifyPublish(make(chan amqp.Confirmation, 1))
+	if err := p.ch.PublishWithContext(ctx, p.exchange, p.routingKey, false, false, amqp.Publishing{
+		ContentType:  "application/json",
+		Body:         body,
+		DeliveryMode: amqp.Persistent,
+	}); err != nil {
+		return err
+	}
+	select {
+	case confirm := <-confirms:
+		if !confirm.Ack {
+			return fmt.Errorf("publish not confirmed")
+		}
+		return nil
+	case <-ctx.Done():
+		return ctx.Err()
+	}
+}

watcher/internal/scanner/scanner.go

@@ -0,0 +1,144 @@
+package scanner
+
+import (
+	"fmt"
+	"log/slog"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/oklog/ulid/v2"
+)
+
+var allowedExts = map[string]bool{
+	".mp3": true, ".wav": true, ".m4a": true,
+	".ogg": true, ".flac": true, ".webm": true,
+}
+
+type ClaimedFile struct {
+	TaskID   string
+	FilePath string
+	Filename string
+	Size     int64
+}
+
+type Config struct {
+	StorageRoot   string
+	IncomingDir   string
+	ProcessingDir string
+	FailedDir     string
+	StableWindow  time.Duration
+	StableChecks  int
+}
+
+type Scanner struct {
+	cfg Config
+}
+
+func New(cfg Config) *Scanner {
+	return &Scanner{cfg: cfg}
+}
+
+func (s *Scanner) EnsureDirs() error {
+	for _, dir := range []string{s.cfg.IncomingDir, s.cfg.ProcessingDir, s.cfg.FailedDir} {
+		if err := os.MkdirAll(filepath.Join(s.cfg.StorageRoot, dir), 0o755); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (s *Scanner) ScanOnce() ([]ClaimedFile, error) {
+	incoming := filepath.Join(s.cfg.StorageRoot, s.cfg.IncomingDir)
+	entries, err := os.ReadDir(incoming)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return nil, nil
+		}
+		return nil, err
+	}
+
+	var claimed []ClaimedFile
+	for _, e := range entries {
+		if e.IsDir() {
+			continue
+		}
+		name := e.Name()
+		if strings.HasPrefix(name, ".") || strings.HasSuffix(strings.ToLower(name), ".tmp") {
+			continue
+		}
+		ext := strings.ToLower(filepath.Ext(name))
+		if !allowedExts[ext] {
+			continue
+		}
+		src := filepath.Join(incoming, name)
+		if !s.isStable(src) {
+			continue
+		}
+		cf, err := s.claim(src, name, ext)
+		if err != nil {
+			slog.Warn("claim failed", "file", name, "error", err)
+			continue
+		}
+		claimed = append(claimed, cf)
+	}
+	return claimed, nil
+}
+
+func (s *Scanner) isStable(path string) bool {
+	var lastSize int64 = -1
+	for i := 0; i < s.cfg.StableChecks; i++ {
+		info, err := os.Stat(path)
+		if err != nil {
+			return false
+		}
+		size := info.Size()
+		if lastSize >= 0 && size != lastSize {
+			return false
+		}
+		lastSize = size
+		if i < s.cfg.StableChecks-1 {
+			time.Sleep(s.cfg.StableWindow)
+		}
+	}
+	return true
+}
+
+func (s *Scanner) claim(src, originalName, ext string) (ClaimedFile, error) {
+	info, err := os.Stat(src)
+	if err != nil {
+		return ClaimedFile{}, err
+	}
+	taskID := ulid.Make().String()
+	processing := filepath.Join(s.cfg.StorageRoot, s.cfg.ProcessingDir)
+	dst := filepath.Join(processing, taskID+ext)
+	if err := os.Rename(src, dst); err != nil {
+		return ClaimedFile{}, fmt.Errorf("rename: %w", err)
+	}
+	slog.Info("claimed file", "task_id", taskID, "filename", originalName, "path", dst, "size", info.Size())
+	return ClaimedFile{
+		TaskID:   taskID,
+		FilePath: dst,
+		Filename: originalName,
+		Size:     info.Size(),
+	}, nil
+}
+
+func (s *Scanner) RollbackToIncoming(filePath, originalName string) error {
+	incoming := filepath.Join(s.cfg.StorageRoot, s.cfg.IncomingDir)
+	dst := filepath.Join(incoming, originalName)
+	if err := os.Rename(filePath, dst); err != nil {
+		return s.MoveToFailed(filePath, originalName)
+	}
+	return nil
+}
+
+func (s *Scanner) MoveToFailed(filePath, originalName string) error {
+	failed := filepath.Join(s.cfg.StorageRoot, s.cfg.FailedDir)
+	if err := os.MkdirAll(failed, 0o755); err != nil {
+		return err
+	}
+	dst := filepath.Join(failed, originalName)
+	return os.Rename(filePath, dst)
+}