fix: make log alert delivery durable
This commit is contained in:
@@ -2,98 +2,179 @@ package ingest
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.apinb.com/ops/logs/internal/config"
|
||||
"git.apinb.com/ops/logs/internal/impl"
|
||||
"git.apinb.com/ops/logs/internal/models"
|
||||
"gorm.io/gorm"
|
||||
"gorm.io/gorm/clause"
|
||||
)
|
||||
|
||||
const (
|
||||
outboxStatusPending = "pending"
|
||||
outboxStatusRetrying = "retrying"
|
||||
outboxStatusSent = "sent"
|
||||
outboxStatusDead = "dead"
|
||||
outboxStatusPending = "pending"
|
||||
outboxStatusProcessing = "processing"
|
||||
outboxStatusRetrying = "retrying"
|
||||
outboxStatusSent = "sent"
|
||||
outboxStatusDead = "dead"
|
||||
)
|
||||
|
||||
func enqueueAlert(logEventID uint, body AlertReceiveBody) error {
|
||||
func enqueueAlertWithDB(db *gorm.DB, logEventID uint, body AlertReceiveBody) (uint, error) {
|
||||
payload, err := json.Marshal(body)
|
||||
if err != nil {
|
||||
return err
|
||||
return 0, err
|
||||
}
|
||||
return enqueuePayload(logEventID, string(payload))
|
||||
return enqueuePayloadWithDB(db, logEventID, string(payload))
|
||||
}
|
||||
|
||||
func enqueueRawEvent(logEventID uint, body AlertReceiveBody, parseStatus string) error {
|
||||
func enqueueRawEventWithDB(db *gorm.DB, logEventID uint, body AlertReceiveBody, parseStatus string) (uint, error) {
|
||||
payload, err := json.Marshal(buildRawEventIngestBody(body, parseStatus))
|
||||
if err != nil {
|
||||
return err
|
||||
return 0, err
|
||||
}
|
||||
return enqueuePayload(logEventID, string(payload))
|
||||
return enqueuePayloadWithDB(db, logEventID, string(payload))
|
||||
}
|
||||
|
||||
func enqueuePayload(logEventID uint, payloadJSON string) error {
|
||||
func enqueuePayloadWithDB(db *gorm.DB, logEventID uint, payloadJSON string) (uint, error) {
|
||||
if db == nil {
|
||||
return 0, fmt.Errorf("database is not initialized")
|
||||
}
|
||||
now, err := databaseNow(db)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
row := models.AlertOutbox{
|
||||
LogEventID: logEventID,
|
||||
PayloadJSON: payloadJSON,
|
||||
Status: outboxStatusPending,
|
||||
RetryCount: 0,
|
||||
NextRetryAt: time.Now(),
|
||||
NextRetryAt: now,
|
||||
LastError: "",
|
||||
}
|
||||
return impl.DBService.Create(&row).Error
|
||||
if err := db.Create(&row).Error; err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return row.ID, nil
|
||||
}
|
||||
|
||||
func StartAlertDispatcher() {
|
||||
owner := dispatcherOwner()
|
||||
go func() {
|
||||
ticker := time.NewTicker(2 * time.Second)
|
||||
defer ticker.Stop()
|
||||
for range ticker.C {
|
||||
processAlertOutboxBatch(20)
|
||||
if _, err := ProcessAlertOutboxBatch(impl.DBService, 20, owner); err != nil {
|
||||
log.Printf("logs: alert outbox dispatch: %v", err)
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
func processAlertOutboxBatch(limit int) {
|
||||
func dispatcherOwner() string {
|
||||
host, _ := os.Hostname()
|
||||
return fmt.Sprintf("%s:%d:%d", host, os.Getpid(), time.Now().UnixNano())
|
||||
}
|
||||
|
||||
// ProcessAlertOutboxBatch 原子领取并处理一批任务，可安全用于多实例 worker。
|
||||
func ProcessAlertOutboxBatch(db *gorm.DB, limit int, owner string) (int, error) {
|
||||
if db == nil {
|
||||
return 0, fmt.Errorf("database is not initialized")
|
||||
}
|
||||
if limit <= 0 {
|
||||
limit = 20
|
||||
}
|
||||
var rows []models.AlertOutbox
|
||||
now := time.Now()
|
||||
err := impl.DBService.
|
||||
Where("status IN ? AND next_retry_at <= ?", []string{outboxStatusPending, outboxStatusRetrying}, now).
|
||||
Order("id asc").
|
||||
Limit(limit).
|
||||
Find(&rows).Error
|
||||
if err != nil || len(rows) == 0 {
|
||||
return
|
||||
owner = strings.TrimSpace(owner)
|
||||
if owner == "" {
|
||||
return 0, fmt.Errorf("outbox lease owner is required")
|
||||
}
|
||||
for _, row := range rows {
|
||||
processOneOutbox(row)
|
||||
processed := 0
|
||||
for processed < limit {
|
||||
rows, err := claimAlertOutboxBatch(db, 1, owner)
|
||||
if err != nil {
|
||||
return processed, err
|
||||
}
|
||||
if len(rows) == 0 {
|
||||
break
|
||||
}
|
||||
if err := processOneOutbox(db, rows[0]); err != nil {
|
||||
return processed, err
|
||||
}
|
||||
processed++
|
||||
}
|
||||
return processed, nil
|
||||
}
|
||||
|
||||
func processOneOutbox(row models.AlertOutbox) {
|
||||
func claimAlertOutboxBatch(db *gorm.DB, limit int, owner string) ([]models.AlertOutbox, error) {
|
||||
var rows []models.AlertOutbox
|
||||
now, err := databaseNow(db)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
leaseUntil := now.Add(30 * time.Second)
|
||||
err = db.Transaction(func(tx *gorm.DB) error {
|
||||
if err := tx.Clauses(clause.Locking{Strength: "UPDATE", Options: "SKIP LOCKED"}).
|
||||
Where("(status IN ? AND next_retry_at <= ?) OR (status = ? AND lease_until IS NOT NULL AND lease_until <= ?)", []string{outboxStatusPending, outboxStatusRetrying}, now, outboxStatusProcessing, now).
|
||||
Order("id asc").Limit(limit).Find(&rows).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
if len(rows) == 0 {
|
||||
return nil
|
||||
}
|
||||
ids := make([]uint, 0, len(rows))
|
||||
for i := range rows {
|
||||
ids = append(ids, rows[i].ID)
|
||||
}
|
||||
if err := tx.Model(&models.AlertOutbox{}).Where("id IN ?", ids).Updates(map[string]interface{}{
|
||||
"status": outboxStatusProcessing,
|
||||
"lease_until": leaseUntil,
|
||||
"lease_owner": owner,
|
||||
}).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
for i := range rows {
|
||||
rows[i].Status = outboxStatusProcessing
|
||||
rows[i].LeaseUntil = &leaseUntil
|
||||
rows[i].LeaseOwner = owner
|
||||
}
|
||||
return nil
|
||||
})
|
||||
return rows, err
|
||||
}
|
||||
|
||||
func processOneOutbox(db *gorm.DB, row models.AlertOutbox) error {
|
||||
var body AlertReceiveBody
|
||||
if err := json.Unmarshal([]byte(row.PayloadJSON), &body); err != nil {
|
||||
markOutboxDead(row.ID, row.RetryCount, "invalid_payload: "+err.Error())
|
||||
return
|
||||
now, nowErr := databaseNow(db)
|
||||
if nowErr != nil {
|
||||
return nowErr
|
||||
}
|
||||
return markOutboxDead(db, row, row.RetryCount, "invalid_payload: "+err.Error(), now)
|
||||
}
|
||||
if err := forwardOutboxPayload(row.PayloadJSON, body); err != nil {
|
||||
markOutboxRetry(row, err.Error())
|
||||
return
|
||||
forwardErr := forwardOutboxPayload(row.PayloadJSON, body)
|
||||
now, err := databaseNow(db)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_ = impl.DBService.Model(&models.AlertOutbox{}).Where("id = ?", row.ID).Updates(map[string]interface{}{
|
||||
"status": outboxStatusSent,
|
||||
"last_error": "",
|
||||
"next_retry_at": time.Now(),
|
||||
}).Error
|
||||
_ = impl.DBService.Model(&models.LogEvent{}).Where("id = ?", row.LogEventID).Updates(map[string]interface{}{
|
||||
"alert_sent": true,
|
||||
"dispatch_status": "sent",
|
||||
}).Error
|
||||
if forwardErr != nil {
|
||||
if errors.Is(forwardErr, errAlertForwardDisabled) {
|
||||
return markOutboxDead(db, row, row.RetryCount, forwardErr.Error(), now)
|
||||
}
|
||||
return markOutboxRetry(db, row, forwardErr.Error(), now)
|
||||
}
|
||||
return markOutboxSent(db, row, now)
|
||||
}
|
||||
|
||||
func databaseNow(db *gorm.DB) (time.Time, error) {
|
||||
var now time.Time
|
||||
if err := db.Raw("SELECT CURRENT_TIMESTAMP").Scan(&now).Error; err != nil {
|
||||
return time.Time{}, err
|
||||
}
|
||||
return now.UTC(), nil
|
||||
}
|
||||
|
||||
func forwardOutboxPayload(payloadJSON string, legacyBody AlertReceiveBody) error {
|
||||
@@ -107,7 +188,7 @@ func forwardOutboxPayload(payloadJSON string, legacyBody AlertReceiveBody) error
|
||||
func forwardRawEvent(body RawEventIngestBody) error {
|
||||
cfg := config.Spec.AlertForward
|
||||
if cfg == nil || !cfg.Enabled || cfg.BaseURL == "" {
|
||||
return nil
|
||||
return errAlertForwardDisabled
|
||||
}
|
||||
if len(body.RawPayload) == 0 {
|
||||
return fmt.Errorf("raw_payload 不能为空")
|
||||
@@ -116,46 +197,103 @@ func forwardRawEvent(body RawEventIngestBody) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return postAlertPayload(cfg, "/Alert/v1/raw-events/ingest", raw)
|
||||
result, err := postAlertPayload(cfg, "/Alert/v1/raw-events/ingest", raw)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var details struct {
|
||||
ID uint `json:"id"`
|
||||
RawEventID uint `json:"raw_event_id"`
|
||||
AlertRecordID uint `json:"alert_record_id"`
|
||||
IncidentID uint `json:"incident_id"`
|
||||
ParseStatus string `json:"parse_status"`
|
||||
Status string `json:"status"`
|
||||
Fingerprint string `json:"fingerprint"`
|
||||
}
|
||||
if err := json.Unmarshal(result.Details, &details); err != nil {
|
||||
return fmt.Errorf("Alert 原始事件响应 details 无效：%v；请稍后重试", err)
|
||||
}
|
||||
if body.ParseStatus == "unparsed" {
|
||||
if details.ID == 0 || (details.ParseStatus != "unparsed" && details.ParseStatus != "replayed") {
|
||||
return fmt.Errorf("Alert 原始事件响应缺少持久化结果，无法确认转发成功；请稍后重试")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
rawEventID := details.RawEventID
|
||||
if rawEventID == 0 {
|
||||
rawEventID = details.ID
|
||||
}
|
||||
if rawEventID == 0 || details.AlertRecordID == 0 || details.IncidentID == 0 ||
|
||||
(details.ID != 0 && details.RawEventID != 0 && details.ID != details.RawEventID) ||
|
||||
(details.Status != "firing" && details.Status != "resolved") || !validFingerprint(details.Fingerprint) {
|
||||
return fmt.Errorf("Alert 原始事件响应缺少完整处理结果，无法确认转发成功；请稍后重试")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func markOutboxRetry(row models.AlertOutbox, msg string) {
|
||||
func markOutboxSent(db *gorm.DB, row models.AlertOutbox, now time.Time) error {
|
||||
return db.Transaction(func(tx *gorm.DB) error {
|
||||
result := tx.Model(&models.AlertOutbox{}).
|
||||
Where("id = ? AND status = ? AND lease_owner = ?", row.ID, outboxStatusProcessing, row.LeaseOwner).
|
||||
Updates(map[string]interface{}{"status": outboxStatusSent, "last_error": "", "next_retry_at": now, "lease_until": nil, "lease_owner": ""})
|
||||
if result.Error != nil {
|
||||
return result.Error
|
||||
}
|
||||
if result.RowsAffected == 0 {
|
||||
return nil
|
||||
}
|
||||
return tx.Model(&models.LogEvent{}).Where("id = ? AND dispatch_outbox_id = ?", row.LogEventID, row.ID).Updates(map[string]interface{}{
|
||||
"alert_sent": true, "dispatch_status": "sent",
|
||||
}).Error
|
||||
})
|
||||
}
|
||||
|
||||
func markOutboxRetry(db *gorm.DB, row models.AlertOutbox, msg string, now time.Time) error {
|
||||
retry := row.RetryCount + 1
|
||||
const maxRetry = 5
|
||||
if retry > maxRetry {
|
||||
markOutboxDead(row.ID, retry, msg)
|
||||
return
|
||||
return markOutboxDead(db, row, retry, msg, now)
|
||||
}
|
||||
backoff := time.Duration(retry*retry) * time.Second
|
||||
if backoff > 60*time.Second {
|
||||
backoff = 60 * time.Second
|
||||
}
|
||||
_ = impl.DBService.Model(&models.AlertOutbox{}).Where("id = ?", row.ID).Updates(map[string]interface{}{
|
||||
"status": outboxStatusRetrying,
|
||||
"retry_count": retry,
|
||||
"next_retry_at": time.Now().Add(backoff),
|
||||
"last_error": truncateError(msg, 1024),
|
||||
}).Error
|
||||
_ = impl.DBService.Model(&models.LogEvent{}).Where("id = ?", row.LogEventID).Update("dispatch_status", "retrying").Error
|
||||
return updateClaimedOutbox(db, row, map[string]interface{}{
|
||||
"status": outboxStatusRetrying, "retry_count": retry, "next_retry_at": now.Add(backoff),
|
||||
"last_error": truncateError(msg, 1024), "lease_until": nil, "lease_owner": "",
|
||||
}, "retrying")
|
||||
}
|
||||
|
||||
func markOutboxDead(id uint, retry int, msg string) {
|
||||
_ = impl.DBService.Model(&models.AlertOutbox{}).Where("id = ?", id).Updates(map[string]interface{}{
|
||||
"status": outboxStatusDead,
|
||||
"retry_count": retry,
|
||||
"next_retry_at": time.Now(),
|
||||
"last_error": truncateError(msg, 1024),
|
||||
}).Error
|
||||
var row models.AlertOutbox
|
||||
if err := impl.DBService.Select("log_event_id").First(&row, id).Error; err == nil && row.LogEventID > 0 {
|
||||
_ = impl.DBService.Model(&models.LogEvent{}).Where("id = ?", row.LogEventID).Update("dispatch_status", "dead").Error
|
||||
}
|
||||
func markOutboxDead(db *gorm.DB, row models.AlertOutbox, retry int, msg string, now time.Time) error {
|
||||
return updateClaimedOutbox(db, row, map[string]interface{}{
|
||||
"status": outboxStatusDead, "retry_count": retry, "next_retry_at": now,
|
||||
"last_error": truncateError(msg, 1024), "lease_until": nil, "lease_owner": "",
|
||||
}, "dead")
|
||||
}
|
||||
|
||||
func updateClaimedOutbox(db *gorm.DB, row models.AlertOutbox, updates map[string]interface{}, eventStatus string) error {
|
||||
return db.Transaction(func(tx *gorm.DB) error {
|
||||
result := tx.Model(&models.AlertOutbox{}).
|
||||
Where("id = ? AND status = ? AND lease_owner = ?", row.ID, outboxStatusProcessing, row.LeaseOwner).
|
||||
Updates(updates)
|
||||
if result.Error != nil {
|
||||
return result.Error
|
||||
}
|
||||
if result.RowsAffected == 0 {
|
||||
return nil
|
||||
}
|
||||
return tx.Model(&models.LogEvent{}).Where("id = ? AND dispatch_outbox_id = ?", row.LogEventID, row.ID).Update("dispatch_status", eventStatus).Error
|
||||
})
|
||||
}
|
||||
|
||||
func truncateError(s string, n int) string {
|
||||
s = strings.TrimSpace(s)
|
||||
if len(s) <= n {
|
||||
if n <= 0 {
|
||||
return ""
|
||||
}
|
||||
runes := []rune(s)
|
||||
if len(runes) <= n {
|
||||
return s
|
||||
}
|
||||
return s[:n]
|
||||
return string(runes[:n])
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user