Add retry logic when susbcription fails

Author: mithun-pensando

redfish-exporter/.env

@@ -1,23 +1,20 @@
-UPDATED="2025-01-22"
+UPDATED="2024-09-24"
 DESCRIPTION="Redfish Event Listener/Exporter"
-LISTENER_IP="<Listener_IP>"
-LISTENER_PORT="<PORT>"
-METRICS_PORT="<MERTRICS_PORT>"
+LISTENER_IP="10.11.18.55"
+LISTENER_PORT="9003"
+METRICS_PORT="2112"
 USE_SSL="false"
 CERTFILE="path/to/certfile"
 KEYFILE="path/to/keyfile"
-SLURM_CONTROL_NODE="<SLURM_CONTROL_NODE_IP>"
-#List of '|' seperated reasons for avoiding drain action if there is a match
-SLURM_DRAIN_EXCLUDE_REASON_LIST="reason 1|reason 2"
+SLURM_CONTROL_NODE="10.235.34.47"
+SLURM_DRAIN_EXCLUDE_REASON_LIST="AMD|Pensando|RebootNeeded"
 SLURM_SCONTROL_PATH="/usr/bin/scontrol"
+TLS_TIMEOUT="15"
 
-# Match RAS events received based on severity and '|' seperated list of message fields and perform drain action with the DrainReasonPrefix set as the prefix in the reason 
-# Message can be left empty if it doesn't need to be matched against, in that case only severity is matched
-# only DrainNode action is supported for now
 TRIGGER_EVENTS="[\
-{\"Severity\":\"Critical\",\"Message\":\"message 1|This is a critical test event\",\"Action\":\"DrainNode\", \"DrainReasonPrefix\":\"RebootNeeded\"},\
-{\"Severity\":\"Info\",\"Message\":\"message 3\",\"Action\":\"DrainNode\", \"DrainReasonPrefix\":\"RebootNotNeeded\"},\
-{\"Severity\":\"Warning\",\"Message\":\"message 4|This is a test event message\",\"Action\":\"DrainNode\", \"DrainReasonPrefix\":\"RebootNotNeeded\"}
+{\"Severity\":\"Critical\",\"Message\":\"Image 'UBB_FPGA' is being verified at 'ERoT'|This is an e2e critical test event\",\"Action\":\"DrainNode\", \"DrainReasonPrefix\":\"RebootNeeded\"},\
+{\"Severity\":\"Info\",\"Message\":\"Image 'UBB_FPGA' is being verified at 'ERoT'\",\"Action\":\"DrainNode\", \"DrainReasonPrefix\":\"RebootNotNeeded\"},\
+{\"Severity\":\"Warning\",\"Message\":\"Image 'UBB_FPGA' is being verified at 'ERoT'|This is an e2e test event message\",\"Action\":\"DrainNode\", \"DrainReasonPrefix\":\"RebootNotNeeded\"}
 ]"
 
 # Subscription (v1.5+)
@@ -33,11 +30,11 @@ TRIGGER_EVENTS="[\
 
 # Deprecated <v1.5
 SUBSCRIPTION_PAYLOAD="{\
-\"Destination\":\"http://<Listener_IP:Port>\",\
+\"Destination\":\"http://10.11.18.55:9003\",\
 \"EventTypes\":[\"Alert\"],\
 \"Protocol\":\"Redfish\",\
 \"Context\":\"YourContextData\",\
-\"Oem\":{\"Supermicro\":{\"EnableSubscription\":true}}\
+\"Oem\":{\"Supermicro\": {\"EnableSubscription\": true}}\
 }"
 
 # Config for setting default labels in Prometheus counter metrics.
@@ -46,5 +43,10 @@ PROMETHEUS_CONFIG="{\
 }"
 
 REDFISH_SERVERS="[\
-{\"ip\":\"https://<BMC_IP>\",\"username\":\"<username>\",\"password\":\"<password>\",\"loginType\":\"Session\",\"slurmNode\":\"<nodename\"}
-]"
+{\"ip\":\"https://10.235.37.54\",\"username\":\"ADMIN\",\"password\":\"PHHCJZUHDV\",\"loginType\":\"Session\",\"slurmNode\":\"smc300x-ccs-aus-GPUFCE9\"},
+{\"ip\":\"https://10.235.37.48\",\"username\":\"ADMIN\",\"password\":\"PHHCJZUHDV\",\"loginType\":\"Session\",\"slurmNode\":\"smc300x-ccs-aus-GPUFCE9\"}
+
+]"
+
+REDFISH_SERVERS_COMMON_CONFIG="{\
+\"hostSuffix\":\"ipmi.cluster\",\"username\":\"<username>\",\"password\":\"<password>\"}"

redfish-exporter/config.go

@@ -19,19 +19,23 @@ package main
 import (
 	"crypto/tls"
 	"encoding/json"
+	"fmt"
 	"log"
+	"net"
 	"os"
 	"strconv"
 	"strings"
 
 	"github.com/joho/godotenv"
+	"gopkg.in/yaml.v3"
 )
 
 const (
 	DefaultListenerPort   = "8080"
 	DefaultMetricsPort    = "2112"
 	DefaultUseSSL         = "false"
 	DefaultSeverityConfig = "Fatal,Critical,Informational"
+	NodeDrainPolicyFile   = "nodeDrainPolicy.json"
 )
 
 type Config struct {
@@ -56,11 +60,19 @@ type Config struct {
 	SlurmDrainExcludeStr string
 	SubscriptionPayload  SubscriptionPayload
 	RedfishServers       []RedfishServer
-	TriggerEvents        []TriggerEvent
+	TriggerEvents        map[string]map[string][]EventInfo //map[Severity][MessageRegistry.MessageId][]EventInfo
 	PrometheusConfig     PrometheusConfig
 	context              *tls.Config
 	eventCount           int
 	dataBuffer           []byte
+	TlsTimeOut           string
+}
+
+type EventInfo struct {
+	UniqueString      string
+	Category          string
+	Subcategory       string
+	DrainReasonPrefix string
 }
 
 type TriggerEvent struct {
@@ -70,11 +82,27 @@ type TriggerEvent struct {
 	DrainReasonPrefix string `json:"DrainReasonPrefix"`
 }
 
+type TriggerEventsInfo struct {
+	Category          string `json:"Category"`
+	Subcategory       string `json:"Subcategory"`
+	MessageRegistry   string `json:"MessageRegistry"`
+	MessageId         string `json:"MessageId"`
+	UniqueString      string `json:"UniqueString"`
+	Severity          string `json:"Severity"`
+	DrainReasonPrefix string `json:"DrainReasonPrefix"`
+	Enable            bool   `json:"Enable"`
+}
+
 type PrometheusConfig struct {
 	Severity []string `json:"Severity"`
 }
 
-func setupConfig() Config {
+type target struct {
+	Targets []string          `yaml:"targets"`
+	Labels  map[string]string `yaml:"labels"`
+}
+
+func setupConfig(targetFile string) Config {
 	// Load .env file
 	err := godotenv.Load()
 	if err != nil {
@@ -125,20 +153,13 @@ func setupConfig() Config {
 	AppConfig.SlurmUser = os.Getenv("SLURM_USER")
 	AppConfig.SlurmDrainExcludeStr = os.Getenv("SLURM_DRAIN_EXCLUDE_REASON_LIST")
 	AppConfig.SlurmScontrolPath = os.Getenv("SLURM_SCONTROL_PATH")
+	AppConfig.TlsTimeOut = os.Getenv("TLS_TIMEOUT")
 
 	subscriptionPayloadJSON := os.Getenv("SUBSCRIPTION_PAYLOAD")
 	if err := json.Unmarshal([]byte(subscriptionPayloadJSON), &AppConfig.SubscriptionPayload); err != nil {
 		log.Fatalf("Failed to parse SUBSCRIPTION_PAYLOAD: %v", err)
 	}
 
-	triggerEventsJSON := os.Getenv("TRIGGER_EVENTS")
-	if triggerEventsJSON != "" {
-		err = json.Unmarshal([]byte(triggerEventsJSON), &AppConfig.TriggerEvents)
-		if err != nil {
-			log.Fatalf("Failed to unmarshal TRIGGER_EVENTS: %v", err)
-		}
-	}
-
 	prometheusConfigJSON := os.Getenv("PROMETHEUS_CONFIG")
 	if prometheusConfigJSON != "" {
 		err = json.Unmarshal([]byte(prometheusConfigJSON), &AppConfig.PrometheusConfig)
@@ -154,10 +175,113 @@ func setupConfig() Config {
 	redfishServersJSON := os.Getenv("REDFISH_SERVERS")
 	if redfishServersJSON == "" {
 		log.Println("REDFISH_SERVERS environment variable is not set or is empty")
+	} else {
+		if err := json.Unmarshal([]byte(redfishServersJSON), &AppConfig.RedfishServers); err != nil {
+			log.Fatalf("Failed to parse REDFISH_SERVERS: %v", err)
+		}
+	}
+
+	// Read the node drain policy config file
+	nodeDrainPolicyConfig, err := os.ReadFile(NodeDrainPolicyFile)
+
+	if err != nil {
+		log.Fatalf("Failed to read: %v", NodeDrainPolicyFile)
+	}
+
+	triggerEventsInfo := []TriggerEventsInfo{}
+	err = json.Unmarshal(nodeDrainPolicyConfig, &triggerEventsInfo)
+	if err != nil {
+		log.Fatalf("Failed to unmarshal file: %v | err: %v", NodeDrainPolicyFile, err)
+	}
+
+	tInfoMap := map[string]map[string][]EventInfo{}
+
+	for _, evt := range triggerEventsInfo {
+		fmt.Printf("Trigger Event: %+v\n", evt)
+		if evt.Enable != true {
+			continue
+		}
+		eInfo := EventInfo{}
+		eInfo.Category = evt.Category
+		eInfo.Subcategory = evt.Subcategory
+		eInfo.DrainReasonPrefix = evt.DrainReasonPrefix
+		eInfo.UniqueString = evt.UniqueString
+		key := ""
+		if evt.MessageRegistry == "" {
+			key = evt.MessageId
+		} else {
+			key = evt.MessageRegistry + "." + evt.MessageId
+		}
+		if ee, ok := tInfoMap[evt.Severity]; !ok {
+			eInfoMap := map[string][]EventInfo{}
+			eInfoMap[key] = []EventInfo{eInfo}
+			tInfoMap[evt.Severity] = eInfoMap
+		} else {
+			ee[key] = append(ee[key], eInfo)
+		}
+	}
+
+	AppConfig.TriggerEvents = tInfoMap
+
+	for kk, tt := range AppConfig.TriggerEvents {
+		fmt.Println("Severity: ", kk)
+		for kkk, ttt := range tt {
+			fmt.Println("key: ", kkk)
+			fmt.Printf("event: %+v\n", ttt)
+		}
+	}
+
+	// Read and parse the REDFISH_SERVERS_COMMON_CONFIG environment variable
+	redfishServersCommonConfigJSON := os.Getenv("REDFISH_SERVERS_COMMON_CONFIG")
+	if redfishServersCommonConfigJSON == "" {
+		log.Println("redfishServersCommonConfigJSON environment variable is not set or is empty")
+		return AppConfig
+	}
+	redfishServersCommonConfig := RedfishServersCommongConfig{}
+	if err := json.Unmarshal([]byte(redfishServersCommonConfigJSON), &redfishServersCommonConfig); err != nil {
+		log.Fatalf("Failed to parse REDFISH_SERVERS_COMMON_CONFIG: %v", err)
+	}
+
+	if targetFile == "" {
+		log.Println("No target file provided")
 		return AppConfig
 	}
-	if err := json.Unmarshal([]byte(redfishServersJSON), &AppConfig.RedfishServers); err != nil {
-		log.Fatalf("Failed to parse REDFISH_SERVERS: %v", err)
+
+	targetYamlFile, err := os.ReadFile(targetFile)
+
+	if err != nil {
+		log.Fatalf("Failed to read file: %v", targetFile)
+	}
+
+	targets := []target{}
+
+	err = yaml.Unmarshal(targetYamlFile, &targets)
+
+	if err != nil {
+		log.Fatalf("Error parsing target file: %v | err: %v", targetFile, err)
+	}
+
+	for _, t := range targets {
+		log.Println("target: ", t.Targets)
+
+		for _, hostName := range t.Targets {
+			// add this target to Redfish servers
+			server := RedfishServer{}
+			bmcHost := fmt.Sprintf(hostName+".%v", redfishServersCommonConfig.HostSuffix)
+			ips, err := net.LookupIP(bmcHost)
+			if err != nil || len(ips) == 0 {
+				log.Printf("[error] Couldn't get the IP for host: %v | ips: %v | err: %v", bmcHost, ips, err)
+				continue
+			}
+			log.Println("IPs: ", ips)
+
+			server.IP = fmt.Sprintf("https://%v", ips[0])
+			server.LoginType = "Session"
+			server.Username = redfishServersCommonConfig.UserName
+			server.Password = redfishServersCommonConfig.Password
+			server.SlurmNode = hostName
+			AppConfig.RedfishServers = append(AppConfig.RedfishServers, server)
+		}
 	}
 
 	return AppConfig

redfish-exporter/go.mod

@@ -9,6 +9,7 @@ require (
 	github.com/nod-ai/ADA/redfish-exporter v0.0.0-20241002210630-2ef2d1070d90
 	github.com/prometheus/client_golang v1.20.4
 	github.com/stmcginnis/gofish v0.19.0
+	gopkg.in/yaml.v3 v3.0.1
 )
 
 require (

redfish-exporter/go.sum

@@ -26,3 +26,6 @@ golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI=
 golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg=
 google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

redfish-exporter/listener.go

@@ -27,7 +27,6 @@ import (
 	"log"
 	"net"
 	"net/http"
-	"regexp"
 	"strings"
 
 	"github.com/nod-ai/ADA/redfish-exporter/metrics"
@@ -172,6 +171,37 @@ func (s *Server) handleConnection(AppConfig Config, conn net.Conn) {
 	}
 }
 
+func getDrainReasonPrefix(info EventInfo) string {
+	return info.DrainReasonPrefix + ": " + info.Category + ": " + info.Subcategory
+}
+
+func isTriggerEvent(evt Event, config Config) (bool, string) {
+	tInfoMap := config.TriggerEvents
+
+	if eInfoMap, ok := tInfoMap[evt.Severity]; !ok {
+		return false, ""
+	} else {
+		if eInfo, ok1 := eInfoMap[evt.MessageId]; !ok1 {
+			return false, ""
+		} else {
+			if len(eInfo) == 1 {
+				return true, getDrainReasonPrefix(eInfo[0])
+			} else {
+				for _, info := range eInfo {
+					strs := strings.Split(info.UniqueString, "|")
+					for _, str := range strs {
+						if strings.Contains(evt.Message, str) == true {
+							return true, getDrainReasonPrefix(info)
+						}
+					}
+
+				}
+			}
+		}
+	}
+	return false, ""
+}
+
 func (s *Server) processRequest(AppConfig Config, conn net.Conn, req *http.Request, eventCount *int, dataBuffer *[]byte) error {
 	// Extract method, headers, and payload
 	method := req.Method
@@ -218,38 +248,26 @@ func (s *Server) processRequest(AppConfig Config, conn net.Conn, req *http.Reque
 		log.Printf("Message ID: %s", messageId)
 		log.Printf("Message Args: %v", messageArgs)
 		log.Printf("Origin Of Condition: %s", originOfCondition)
-		for _, triggerEvent := range AppConfig.TriggerEvents {
-			if severity == triggerEvent.Severity {
-				if triggerEvent.Message != "" {
-					re := regexp.MustCompile(triggerEvent.Message)
-					match := re.FindAllString(message, -1)
-
-					if len(match) == 0 {
-						continue
-					}
+
+		trigger, drainReason := isTriggerEvent(event, AppConfig)
+		if trigger == true {
+			log.Printf("Matched Trigger Event: %s | messageId: %s | message: %s", event.Severity, event.MessageId, event.Message)
+			// Sending event belongs to redfish_utils. Each server may have different slurm node associated, and redfish_servers has the info/map.
+			if s.slurmQueue != nil {
+				redfishServerInfo := getServerInfoByIP(AppConfig.RedfishServers, ip)
+				if len(strings.TrimSpace(redfishServerInfo.SlurmNode)) == 0 {
+					log.Println("failed to get the slurm node name, cannot perform drain action")
+					continue
 				}
-				log.Printf("Matched Trigger Event: %s | message: %s | with action %s", triggerEvent.Severity, triggerEvent.Message, triggerEvent.Action)
-				// Sending event belongs to redfish_utils. Each server may have different slurm node associated, and redfish_servers has the info/map.
-				if s.slurmQueue != nil {
-					redfishServerInfo := getServerInfoByIP(AppConfig.RedfishServers, ip)
-					if len(strings.TrimSpace(redfishServerInfo.SlurmNode)) == 0 {
-						log.Printf("failed to get the slurm node name, cannot perform action: %v", triggerEvent.Action)
-						break
-					}
-					evt := slurm.AddEventReq{
-						RedfishServerIP:   redfishServerInfo.IP,
-						SlurmNodeName:     redfishServerInfo.SlurmNode,
-						Severity:          triggerEvent.Severity,
-						Action:            triggerEvent.Action,
-						DrainReasonPrefix: triggerEvent.DrainReasonPrefix,
-						MessageId:         messageId,
-						Message:           message,
-						ExcludeStr:        AppConfig.SlurmDrainExcludeStr,
-						ScontrolPath:      AppConfig.SlurmScontrolPath,
-					}
-					s.slurmQueue.Add(evt)
+				evt := slurm.AddEventReq{
+					RedfishServerIP: redfishServerInfo.IP,
+					SlurmNodeName:   redfishServerInfo.SlurmNode,
+					Severity:        event.Severity,
+					DrainReason:     drainReason,
+					ExcludeStr:      AppConfig.SlurmDrainExcludeStr,
+					ScontrolPath:    AppConfig.SlurmScontrolPath,
 				}
-				break
+				s.slurmQueue.Add(evt)
 			}
 		}
 	}