ops/logs
5
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: correlate log alert recovery lifecycle

Browse Source
This commit is contained in:
zxr
2026-07-21 22:15:59 +08:00
parent eaca3d0ac8
commit 8495fdf039
6 changed files with 194 additions and 99 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
internal/ingest/alert_forward.go
View File

@@ -38,6 +38,7 @@ type AlertReceiveBody struct {
Labels map[string]string `json:"labels"`
Agent string `json:"agent"`
PolicyID uint `json:"policy_id"`
Fingerprint string `json:"fingerprint,omitempty"`
State string `json:"state,omitempty"`
SourceEventKey string `json:"source_event_key"`
TraceID string `json:"trace_id"`
@@ -104,6 +105,9 @@ func forwardAlert(body AlertReceiveBody) error {
(details.Status != "firing" && details.Status != "resolved") || !validFingerprint(details.Fingerprint) {
return fmt.Errorf("Alert 响应缺少完整写入结果，无法确认转发成功；请稍后重试")
}
if body.Fingerprint != "" && details.Fingerprint != body.Fingerprint {
return fmt.Errorf("Alert 返回的告警指纹与请求不一致；请稍后重试")
}
return nil
}

151
internal/ingest/engine.go
View File

@@ -1,6 +1,8 @@
package ingest
import (
"crypto/sha256"
"encoding/hex"
"encoding/json"
"fmt"
"log"
@@ -267,16 +269,13 @@ func (e *Engine) HandleSyslog(addr *net.UDPAddr, payload []byte) {
if err != nil {
return 0, err
}
state := "firing"
if isRecoverySignal(parsed.Message, parsed.RawLine) {
state = "resolved"
}
body := AlertReceiveBody{
AlertName: matched.AlertName, Summary: summary, Description: summary,
SeverityCode: firstNonEmpty(matchDetails.SeverityCode, firstNonEmpty(matched.SeverityCode, sev)),
Value: parsed.Message, Labels: labels, Agent: "logs-syslog", PolicyID: matched.PolicyID,
State: state, SourceEventKey: logSourceEventKey(stored, "alert"), OccurredAt: occurredAt, RawData: rawBytes,
State: matchDetails.State, SourceEventKey: logSourceEventKey(stored, "alert"), OccurredAt: occurredAt, RawData: rawBytes,
}
body.Fingerprint = alertLifecycleFingerprint(body, matched.LifecycleKey, "")
return enqueueAlertWithDB(tx, stored.ID, body)
}); err != nil {
log.Printf("logs: persist matched syslog event: %v", err)
@@ -285,6 +284,7 @@ func (e *Engine) HandleSyslog(addr *net.UDPAddr, payload []byte) {
type syslogRuleMatch struct {
Matched bool
State string
ResourceUID string
SeverityCode string
Captures map[string]string
@@ -295,11 +295,12 @@ func syslogRuleMatches(rule *models.SyslogRule, device, message, rawLine string)
}
func syslogRuleMatchDetails(rule *models.SyslogRule, device, message, rawLine string) syslogRuleMatch {
result := syslogRuleMatch{Captures: map[string]string{}}
result := syslogRuleMatch{State: "firing", Captures: map[string]string{}}
deviceContains := strings.TrimSpace(rule.DeviceNameContains)
sourceMatch := strings.TrimSpace(rule.SourceMatch)
keywordRegex := strings.TrimSpace(rule.KeywordRegex)
messageRegex := strings.TrimSpace(rule.MessageRegex)
recoveryRegex := strings.TrimSpace(rule.RecoveryMatchRegex)
if deviceContains == "" && sourceMatch == "" && keywordRegex == "" && messageRegex == "" {
return result
}
@@ -316,6 +317,19 @@ func syslogRuleMatchDetails(rule *models.SyslogRule, device, message, rawLine st
return result
}
}
if recoveryRegex != "" {
re, err := regexp.Compile(recoveryRegex)
if err != nil {
return result
}
matches := firstRegexMatch(re, message, rawLine)
if matches != nil {
mergeNamedCaptures(result.Captures, re, matches)
result.State = "resolved"
result.Matched = true
}
}
if !result.Matched {
for _, pattern := range []string{keywordRegex, messageRegex} {
if pattern == "" {
continue
@@ -324,16 +338,14 @@ func syslogRuleMatchDetails(rule *models.SyslogRule, device, message, rawLine st
if err != nil {
return result
}
matches := re.FindStringSubmatch(message)
if matches == nil {
matches = re.FindStringSubmatch(rawLine)
}
matches := firstRegexMatch(re, message, rawLine)
if matches == nil {
return result
}
mergeNamedCaptures(result.Captures, re, matches)
}
result.Matched = true
}
if uid := extractWithNamedRegex(rule.ResourceUIDExtractRegex, "resource_uid", message, rawLine); uid != "" {
result.ResourceUID = normalizeExtractedResourceUID(uid)
} else if uid := result.Captures["resource_uid"]; uid != "" {
@@ -343,6 +355,15 @@ func syslogRuleMatchDetails(rule *models.SyslogRule, device, message, rawLine st
return result
}
func firstRegexMatch(re *regexp.Regexp, values ...string) []string {
for _, value := range values {
if matches := re.FindStringSubmatch(value); matches != nil {
return matches
}
}
return nil
}
func mergeNamedCaptures(dst map[string]string, re *regexp.Regexp, matches []string) {
names := re.SubexpNames()
for i, name := range names {
@@ -370,7 +391,9 @@ func extractWithNamedRegex(pattern, groupName, message, rawLine string) string {
names := re.SubexpNames()
for i, name := range names {
if i > 0 && name == groupName && i < len(matches) {
return strings.TrimSpace(matches[i])
if value := strings.TrimSpace(matches[i]); value != "" {
return value
}
}
}
for i := 1; i < len(matches); i++ {
@@ -510,8 +533,8 @@ func (e *Engine) HandleTrap(addr *net.UDPAddr, pkt *gosnmp.SnmpPacket) {
rules := e.trapRules
e.mu.RUnlock()
matched := firstMatchingTrapRule(rules, trapOID, fp)
if matched == nil {
match := firstMatchingTrapRule(rules, trapOID, fp)
if match.Rule == nil {
rawBytes, mErr := json.Marshal(fp)
if mErr != nil {
return
@@ -541,6 +564,7 @@ func (e *Engine) HandleTrap(addr *net.UDPAddr, pkt *gosnmp.SnmpPacket) {
}
return
}
matched := match.Rule
desc := readable
if dict != nil && dict.RecoveryMessage != "" {
@@ -555,6 +579,10 @@ func (e *Engine) HandleTrap(addr *net.UDPAddr, pkt *gosnmp.SnmpPacket) {
"instance": addr.IP.String(),
"job": "logs-trap",
}
trapInstance := trapInstanceKey(pkt)
if trapInstance != "" {
labels["trap_instance"] = trapInstance
}
if matched.ID != 0 {
labels["resource_type"] = "trap_rule"
labels["resource_id"] = strconv.FormatUint(uint64(matched.ID), 10)
@@ -588,20 +616,13 @@ func (e *Engine) HandleTrap(addr *net.UDPAddr, pkt *gosnmp.SnmpPacket) {
if err != nil {
return 0, err
}
state := "firing"
dictText := ""
if dict != nil {
dictText = firstNonEmpty(dict.Name, dict.Title)
}
if isRecoverySignal(readable, dictText, matched.Name, matched.AlertName) {
state = "resolved"
}
body := AlertReceiveBody{
AlertName: firstNonEmpty(matched.AlertName, "SNMP Trap"), Summary: readable, Description: desc,
SeverityCode: firstNonEmpty(matched.SeverityCode, sev), Value: string(vbJSON), Labels: labels,
Agent: "logs-trap", PolicyID: matched.PolicyID, State: state,
Agent: "logs-trap", PolicyID: matched.PolicyID, State: match.State,
SourceEventKey: logSourceEventKey(stored, "alert"), OccurredAt: occurredAt, RawData: rawBytes,
}
body.Fingerprint = alertLifecycleFingerprint(body, matched.LifecycleKey, trapInstance)
return enqueueAlertWithDB(tx, stored.ID, body)
}); err != nil {
log.Printf("logs: persist matched trap event: %v", err)
@@ -646,6 +667,27 @@ func trapVarbinds(pkt *gosnmp.SnmpPacket) []map[string]string {
return out
}
func trapInstanceKey(pkt *gosnmp.SnmpPacket) string {
if pkt == nil {
return ""
}
for _, prefix := range []string{
"1.3.6.1.2.1.31.1.1.1.1.",
"1.3.6.1.2.1.2.2.1.2.",
"1.3.6.1.2.1.2.2.1.1.",
} {
for _, variable := range pkt.Variables {
name := normOID(variable.Name)
if strings.HasPrefix(name, prefix) {
if index := strings.TrimSpace(strings.TrimPrefix(name, prefix)); index != "" {
return index
}
}
}
}
return ""
}
func buildTrapReadable(trapOID string, dict *models.TrapDictionaryEntry, varbindSummary string) string {
if dict != nil && firstNonEmpty(dict.Name, dict.Title) != "" {
return firstNonEmpty(dict.Name, dict.Title) + " (" + trapOID + ")"
@@ -656,34 +698,49 @@ func buildTrapReadable(trapOID string, dict *models.TrapDictionaryEntry, varbind
return truncate(varbindSummary, 256)
}
func trapRuleMatches(rule *models.TrapRule, trapOID, varbindFP string) bool {
type trapRuleMatch struct {
Rule *models.TrapRule
State string
}
func trapRuleState(rule *models.TrapRule, trapOID, varbindFP string) (string, bool) {
hasOID := strings.TrimSpace(rule.OIDPrefix) != ""
hasRE := strings.TrimSpace(rule.VarbindMatchRegex) != ""
hasRecoveryRE := strings.TrimSpace(rule.RecoveryMatchRegex) != ""
if !hasOID && !hasRE {
return false
return "", false
}
if hasOID && !strings.HasPrefix(normOID(trapOID), normOID(rule.OIDPrefix)) {
return false
return "", false
}
if hasRecoveryRE {
re, err := regexp.Compile(rule.RecoveryMatchRegex)
if err != nil {
return "", false
}
if re.MatchString(trapOID) || re.MatchString(varbindFP) {
return "resolved", true
}
}
if hasRE {
re, err := regexp.Compile(rule.VarbindMatchRegex)
if err != nil {
return false
return "", false
}
if !re.MatchString(varbindFP) {
return false
return "", false
}
}
return true
return "firing", true
}
func firstMatchingTrapRule(rules []models.TrapRule, trapOID, varbindFP string) *models.TrapRule {
func firstMatchingTrapRule(rules []models.TrapRule, trapOID, varbindFP string) trapRuleMatch {
for i := range rules {
if trapRuleMatches(&rules[i], trapOID, varbindFP) {
return &rules[i]
if state, matched := trapRuleState(&rules[i], trapOID, varbindFP); matched {
return trapRuleMatch{Rule: &rules[i], State: state}
}
}
return nil
return trapRuleMatch{}
}
func firstNonEmpty(a, b string) string {
@@ -693,19 +750,27 @@ func firstNonEmpty(a, b string) string {
return b
}
func isRecoverySignal(values ...string) bool {
for _, value := range values {
text := strings.ToLower(strings.TrimSpace(value))
if text == "" {
continue
func alertLifecycleFingerprint(body AlertReceiveBody, lifecycleKey, instance string) string {
lifecycleKey = strings.TrimSpace(lifecycleKey)
if lifecycleKey == "" {
return ""
}
for _, marker := range []string{"恢复", "recovered", "recovery", "ifup", "link up", "port up", "interface up", "normal"} {
if strings.Contains(text, marker) {
return true
resourceUID := ""
sourceIP := ""
if body.Labels != nil {
resourceUID = strings.TrimSpace(body.Labels["resource_uid"])
sourceIP = strings.TrimSpace(body.Labels["ip"])
}
}
}
return false
identity := strings.Join([]string{
strings.TrimSpace(body.Agent),
resourceUID,
sourceIP,
strconv.FormatUint(uint64(body.PolicyID), 10),
lifecycleKey,
strings.TrimSpace(instance),
}, "\x00")
sum := sha256.Sum256([]byte(identity))
return hex.EncodeToString(sum[:])
}
func (e *Engine) resolveResource(sourceIP, hostname string) (resourceRef, string) {

14
internal/logic/controllers/rule_validation.go
View File

@@ -16,6 +16,7 @@ func validateSyslogRule(rule *models.SyslogRule) error {
}{
{name: "keyword_regex", pattern: rule.KeywordRegex},
{name: "message_regex", pattern: rule.MessageRegex},
{name: "recovery_match_regex", pattern: rule.RecoveryMatchRegex},
{name: "resource_uid_extract_regex", pattern: rule.ResourceUIDExtractRegex},
}
for _, field := range regexFields {
@@ -48,18 +49,25 @@ func validateSyslogRule(rule *models.SyslogRule) error {
strings.TrimSpace(rule.MessageRegex) == "" {
return fmt.Errorf("Syslog 规则的匹配条件全部为空，运行时永远不会命中；请至少填写 device_name_contains、source_match、keyword_regex、message_regex 中的一项")
}
if strings.TrimSpace(rule.RecoveryMatchRegex) != "" && strings.TrimSpace(rule.LifecycleKey) == "" {
return fmt.Errorf("配置 recovery_match_regex 时 lifecycle_key 不能为空")
}
return nil
}
func validateTrapRule(rule *models.TrapRule) error {
if strings.TrimSpace(rule.VarbindMatchRegex) != "" {
if _, err := regexp.Compile(rule.VarbindMatchRegex); err != nil {
return fmt.Errorf("varbind_match_regex 不是有效正则表达式：%v；请修正 varbind_match_regex 后重试", err)
if err := validateOptionalRegex("varbind_match_regex", rule.VarbindMatchRegex); err != nil {
return err
}
if err := validateOptionalRegex("recovery_match_regex", rule.RecoveryMatchRegex); err != nil {
return err
}
if strings.TrimSpace(rule.OIDPrefix) == "" && strings.TrimSpace(rule.VarbindMatchRegex) == "" {
return fmt.Errorf("Trap 规则的匹配条件全部为空，运行时永远不会命中；请至少填写 oid_prefix、varbind_match_regex 中的一项")
}
if strings.TrimSpace(rule.RecoveryMatchRegex) != "" && strings.TrimSpace(rule.LifecycleKey) == "" {
return fmt.Errorf("配置 recovery_match_regex 时 lifecycle_key 不能为空")
}
return nil
}

14
internal/models/query.go
View File

@@ -108,6 +108,8 @@ func seedDefaultSyslogRules(db *gorm.DB) error {
KeywordRegex: "(?i)(link down|interface .* down|port .* down)",
SourceMatch: "",
MessageRegex: "(?i)(link down|interface .* down|port .* down|LINK_DOWN)",
RecoveryMatchRegex: `(?i)(link[ _-]?up|interface .* up|port .* up|ifup)`,
LifecycleKey: "syslog-link-state",
AlertName: "Syslog链路中断",
SeverityCode: "major",
SeverityMappingJSON: `{"(?i)(critical|fatal|emergency)":"critical","(?i)(error|LINK_DOWN|down)":"major","(?i)(warning|warn)":"warning"}`,
@@ -118,8 +120,10 @@ func seedDefaultSyslogRules(db *gorm.DB) error {
Name: "H3C-Syslog-接口中断",
Enabled: true,
Priority: 120,
SourceMatch: "h3c",
DeviceNameContains: "h3c",
MessageRegex: `(?i)(LINK_DOWN|Interface .* down|port .* down)`,
RecoveryMatchRegex: `(?i)(link[ _-]?up|interface .* up|port .* up|ifup)`,
LifecycleKey: "h3c-syslog-interface-state",
AlertName: "H3C Syslog接口中断",
SeverityCode: "major",
SeverityMappingJSON: `{"(?i)(LINK_DOWN|down)":"major","(?i)(LINK_UP|up)":"info"}`,
@@ -147,6 +151,8 @@ func seedDefaultSyslogRules(db *gorm.DB) error {
"source_match",
"keyword_regex",
"message_regex",
"recovery_match_regex",
"lifecycle_key",
"alert_name",
"severity_code",
"severity_mapping_json",
@@ -166,7 +172,9 @@ func seedDefaultTrapRules(db *gorm.DB) error {
Enabled: true,
Priority: 100,
OIDPrefix: "1.3.6.1.6.3.1.1.5",
VarbindMatchRegex: "(?i)(linkdown|ifdown|down)",
VarbindMatchRegex: `(?i)(1\.3\.6\.1\.6\.3\.1\.1\.5\.3([^0-9]|$)|\b(linkdown|ifdown|down)\b)`,
RecoveryMatchRegex: `(?i)(1\.3\.6\.1\.6\.3\.1\.1\.5\.4([^0-9]|$)|\b(linkup|ifup)\b)`,
LifecycleKey: "snmp-interface-link-state",
AlertName: "SNMP Trap链路中断",
SeverityCode: "major",
PolicyID: 0,
@@ -190,6 +198,8 @@ func seedDefaultTrapRules(db *gorm.DB) error {
"priority",
"o_id_prefix",
"varbind_match_regex",
"recovery_match_regex",
"lifecycle_key",
"alert_name",
"severity_code",
"policy_id",

4
internal/models/syslog_rule.go
View File

@@ -24,6 +24,10 @@ type SyslogRule struct {
KeywordRegex string `gorm:"size:512" json:"keyword_regex"`
// MessageRegex 表示消息正文匹配的正则表达式。
MessageRegex string `gorm:"size:1024" json:"message_regex"`
// RecoveryMatchRegex 匹配同一生命周期的恢复消息。
RecoveryMatchRegex string `gorm:"size:1024" json:"recovery_match_regex"`
// LifecycleKey 将故障和恢复事件绑定到同一告警生命周期。
LifecycleKey string `gorm:"size:256" json:"lifecycle_key"`
// AlertName 表示告警名称。
AlertName string `gorm:"size:256" json:"alert_name"`
// SeverityCode 表示严重级别编码。

4
internal/models/trap_rule.go
View File

@@ -20,6 +20,10 @@ type TrapRule struct {
OIDPrefix string `gorm:"size:512" json:"oid_prefix"`
// VarbindMatchRegex 表示对 varbind 内容的正则匹配条件。
VarbindMatchRegex string `gorm:"size:512" json:"varbind_match_regex"`
// RecoveryMatchRegex 匹配同一生命周期的恢复 Trap OID 或 varbind。
RecoveryMatchRegex string `gorm:"size:1024" json:"recovery_match_regex"`
// LifecycleKey 将故障和恢复事件绑定到同一告警生命周期。
LifecycleKey string `gorm:"size:256" json:"lifecycle_key"`
// AlertName 表示告警名称。
AlertName string `gorm:"size:256" json:"alert_name"`
// SeverityCode 表示严重级别编码。