feat: schedule dataset collection

backend/internal/crontab/dataset.go

@@ -0,0 +1,89 @@
+package crontab
+
+import (
+	"context"
+	"log"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"senlinai-agent/backend/internal/models"
+)
+
+const datasetInterval = 5 * time.Minute
+
+type DatasetCollector interface {
+	SyncAllSources(context.Context) ([]models.SaDatasetCron, error)
+}
+
+type DatasetScheduler struct {
+	collector DatasetCollector
+	interval  time.Duration
+	logger    *log.Logger
+}
+
+func NewDatasetScheduler(collector DatasetCollector, logger *log.Logger) *DatasetScheduler {
+	if logger == nil {
+		logger = log.Default()
+	}
+	return &DatasetScheduler{
+		collector: collector,
+		interval:  datasetInterval,
+		logger:    logger,
+	}
+}
+
+// Run collects once at startup, then on a fixed five-minute interval.
+// A tick is skipped when the previous collection is still running.
+func (s *DatasetScheduler) Run(ctx context.Context) {
+	if ctx.Err() != nil {
+		return
+	}
+	ticker := time.NewTicker(s.interval)
+	defer ticker.Stop()
+
+	var running atomic.Bool
+	var workers sync.WaitGroup
+	start := func() {
+		if !running.CompareAndSwap(false, true) {
+			s.logger.Print("dataset collection skipped: previous run is still active")
+			return
+		}
+		workers.Add(1)
+		go func() {
+			defer workers.Done()
+			defer running.Store(false)
+			s.collect(ctx)
+		}()
+	}
+	start()
+
+	for {
+		select {
+		case <-ctx.Done():
+			workers.Wait()
+			return
+		case <-ticker.C:
+			start()
+		}
+	}
+}
+
+func (s *DatasetScheduler) collect(ctx context.Context) {
+	crons, err := s.collector.SyncAllSources(ctx)
+	completed := 0
+	failed := 0
+	for _, cron := range crons {
+		switch cron.Status {
+		case "completed":
+			completed++
+		case "failed":
+			failed++
+		}
+	}
+	if err != nil {
+		s.logger.Printf("dataset collection failed: tasks=%d completed=%d failed=%d error=%v", len(crons), completed, failed, err)
+		return
+	}
+	s.logger.Printf("dataset collection finished: tasks=%d completed=%d failed=%d", len(crons), completed, failed)
+}

backend/internal/crontab/dataset_test.go

@@ -0,0 +1,150 @@
+package crontab
+
+import (
+	"context"
+	"io"
+	"log"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+	"senlinai-agent/backend/internal/models"
+)
+
+func TestDatasetSchedulerCollectsImmediatelyAndRepeats(t *testing.T) {
+	collector := &recordingDatasetCollector{calls: make(chan time.Time, 3)}
+	scheduler := &DatasetScheduler{
+		collector: collector,
+		interval:  10 * time.Millisecond,
+		logger:    log.New(io.Discard, "", 0),
+	}
+	ctx, cancel := context.WithCancel(context.Background())
+	done := make(chan struct{})
+	go func() {
+		defer close(done)
+		scheduler.Run(ctx)
+	}()
+
+	first := receiveCollectionCall(t, collector.calls)
+	second := receiveCollectionCall(t, collector.calls)
+	require.GreaterOrEqual(t, second.Sub(first), 8*time.Millisecond)
+
+	cancel()
+	require.Eventually(t, func() bool {
+		select {
+		case <-done:
+			return true
+		default:
+			return false
+		}
+	}, time.Second, 5*time.Millisecond)
+}
+
+func TestDatasetSchedulerStopsWhenContextIsAlreadyCanceled(t *testing.T) {
+	collector := &recordingDatasetCollector{calls: make(chan time.Time, 1)}
+	scheduler := &DatasetScheduler{
+		collector: collector,
+		interval:  10 * time.Millisecond,
+		logger:    log.New(io.Discard, "", 0),
+	}
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel()
+
+	scheduler.Run(ctx)
+
+	require.Zero(t, collector.callCount())
+}
+
+func TestDatasetSchedulerSkipsTicksWhileCollectionIsRunning(t *testing.T) {
+	collector := &blockingDatasetCollector{
+		started: make(chan struct{}, 2),
+		release: make(chan struct{}),
+	}
+	scheduler := &DatasetScheduler{
+		collector: collector,
+		interval:  5 * time.Millisecond,
+		logger:    log.New(io.Discard, "", 0),
+	}
+	ctx, cancel := context.WithCancel(context.Background())
+	done := make(chan struct{})
+	go func() {
+		defer close(done)
+		scheduler.Run(ctx)
+	}()
+
+	select {
+	case <-collector.started:
+	case <-time.After(time.Second):
+		t.Fatal("timed out waiting for first collection")
+	}
+	time.Sleep(20 * time.Millisecond)
+	require.Equal(t, 1, collector.callCount())
+
+	close(collector.release)
+	require.Eventually(t, func() bool {
+		return collector.callCount() >= 2
+	}, time.Second, 5*time.Millisecond)
+	cancel()
+	<-done
+}
+
+type recordingDatasetCollector struct {
+	mu    sync.Mutex
+	count int
+	calls chan time.Time
+}
+
+func (c *recordingDatasetCollector) SyncAllSources(context.Context) ([]models.SaDatasetCron, error) {
+	now := time.Now()
+	c.mu.Lock()
+	c.count++
+	c.mu.Unlock()
+	c.calls <- now
+	return []models.SaDatasetCron{{Status: "completed"}}, nil
+}
+
+func (c *recordingDatasetCollector) callCount() int {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	return c.count
+}
+
+func receiveCollectionCall(t *testing.T, calls <-chan time.Time) time.Time {
+	t.Helper()
+	select {
+	case calledAt := <-calls:
+		return calledAt
+	case <-time.After(time.Second):
+		t.Fatal("timed out waiting for dataset collection")
+		return time.Time{}
+	}
+}
+
+type blockingDatasetCollector struct {
+	mu      sync.Mutex
+	count   int
+	started chan struct{}
+	release chan struct{}
+}
+
+func (c *blockingDatasetCollector) SyncAllSources(ctx context.Context) ([]models.SaDatasetCron, error) {
+	c.mu.Lock()
+	c.count++
+	c.mu.Unlock()
+	select {
+	case c.started <- struct{}{}:
+	default:
+	}
+	select {
+	case <-c.release:
+	case <-ctx.Done():
+	}
+	return nil, nil
+}
+
+func (c *blockingDatasetCollector) callCount() int {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	return c.count
+}