feat:device add raw rate & FDR effective lane rate accurate to 13.64 (#…

…16) * feat:device add raw rate & FDR effective lane rate accurate to 13.64 * chore: add raw rate metric describe * perf(ibnetdiscoverParse): name may have space so that it is split into multiple items * perf(ibnetdiscoverParse): name may have space so that it is split into multiple items * fix: test * fix:test
treydock · Dec 3, 2023 · 60572b5 · 60572b5
1 parent 49299f5
commit 60572b5
Show file tree

Hide file tree

Showing 10 changed files with 204 additions and 81 deletions.
diff --git a/collectors/collectors_test.go b/collectors/collectors_test.go
@@ -30,12 +30,12 @@ var (
  mockedStdout string
  _, cancel = context.WithTimeout(context.Background(), 5*time.Second)
  switchDevices = []InfinibandDevice{
- {Type: "SW", LID: "2052", GUID: "0x506b4b03005c2740", Rate: (25 * 4 * 125000000), Name: "ib-i4l1s01",
+ {Type: "SW", LID: "2052", GUID: "0x506b4b03005c2740", Rate: (25 * 4 * 125000000), RawRate: 1.2890625e+10, Name: "ib-i4l1s01",
  Uplinks: map[string]InfinibandUplink{
  "35": {Type: "CA", LID: "1432", PortNumber: "1", GUID: "0x506b4b0300cc02a6", Name: "p0001"},
  },
  },
- {Type: "SW", LID: "1719", GUID: "0x7cfe9003009ce5b0", Rate: (25 * 4 * 125000000), Name: "ib-i1l1s01",
+ {Type: "SW", LID: "1719", GUID: "0x7cfe9003009ce5b0", Rate: (25 * 4 * 125000000), RawRate: 1.2890625e+10, Name: "ib-i1l1s01",
  Uplinks: map[string]InfinibandUplink{
  "1": {Type: "SW", LID: "1516", PortNumber: "1", GUID: "0x7cfe900300b07320", Name: "ib-i1l2s01"},
  "10": {Type: "CA", LID: "134", PortNumber: "1", GUID: "0x7cfe9003003b4bde", Name: "o0001"},

diff --git a/collectors/fixtures/ibnetdiscover/test2.out b/collectors/fixtures/ibnetdiscover/test2.out
@@ -0,0 +1,5 @@
+CA 78 1 0x946dae0300630bfe 4x HDR - SW 51 81 0x946dae0300630bf6 ( 'Mellanox Technologies Aggregation Node' - '5FB0405-leaf-IB01 ' )
+CA 88 1 0xb83fd20300da1138 4x HDR - SW 51 79 0x946dae0300630bf6 ( 'worker20 mlx5_3' - '5FB0405-leaf-IB01 ' )
+SW 24 1 0x946dae0300618c82 4x SDR '5FB0406-spine-IB01 '
+SW 25 1 0x946dae0300618c83 4x SDR ' 5FB0406-spine-IB02 '
+SW 9 81 0x946dae030053ec1a 4x HDR - CA 60 1 0x946dae0300630bfe ( ' 5FB0406-spine-IB03 ' - 'Mellanox Technologies Aggregation Node' )
diff --git a/collectors/hca.go b/collectors/hca.go
@@ -65,6 +65,7 @@ type HCACollector struct {
  PortVLMappingErrors *prometheus.Desc
  PortLoopingErrors *prometheus.Desc
  Rate *prometheus.Desc
+ RawRate *prometheus.Desc
  Uplink *prometheus.Desc
  Info *prometheus.Desc
 }
@@ -137,6 +138,8 @@ func NewHCACollector(devices *[]InfinibandDevice, runonce bool, logger log.Logge
  "Infiniband HCA port PortLoopingErrors", labels, nil),
  Rate: prometheus.NewDesc(prometheus.BuildFQName(namespace, "hca", "rate_bytes_per_second"),
  "Infiniband HCA rate", []string{"guid"}, nil),
+ RawRate: prometheus.NewDesc(prometheus.BuildFQName(namespace, "hca", "raw_rate_bytes_per_second"),
+ "Infiniband HCA raw rate", []string{"guid"}, nil),
  Uplink: prometheus.NewDesc(prometheus.BuildFQName(namespace, "hca", "uplink_info"),
  "Infiniband HCA uplink information", append(labels, []string{"hca", "uplink", "uplink_guid", "uplink_type", "uplink_port", "uplink_lid"}...), nil),
  Info: prometheus.NewDesc(prometheus.BuildFQName(namespace, "hca", "info"),
@@ -174,6 +177,7 @@ func (h *HCACollector) Describe(ch chan<- *prometheus.Desc) {
  ch <- h.PortVLMappingErrors
  ch <- h.PortLoopingErrors
  ch <- h.Rate
+ ch <- h.RawRate
  ch <- h.Uplink
  ch <- h.Info
 }
@@ -270,6 +274,7 @@ func (h *HCACollector) Collect(ch chan<- prometheus.Metric) {
  if *hcaCollectBase {
  for _, device := range *h.devices {
  ch <- prometheus.MustNewConstMetric(h.Rate, prometheus.GaugeValue, device.Rate, device.GUID)
+ ch <- prometheus.MustNewConstMetric(h.RawRate, prometheus.GaugeValue, device.RawRate, device.GUID)
  ch <- prometheus.MustNewConstMetric(h.Info, prometheus.GaugeValue, 1, device.GUID, device.Name, device.LID)
  for port, uplink := range device.Uplinks {
  ch <- prometheus.MustNewConstMetric(h.Uplink, prometheus.GaugeValue, 1, device.GUID, port, device.Name, uplink.Name, uplink.GUID, uplink.Type, uplink.PortNumber, uplink.LID)
@@ -294,7 +299,10 @@ func (h *HCACollector) collect() ([]PerfQueryCounters, float64, float64) {
  limit <- 1
  wg.Add(1)
  go func(device InfinibandDevice) {
- defer wg.Done()
+ defer func() {
+ <-limit
+ wg.Done()
+ }()
  ctxExtended, cancelExtended := context.WithTimeout(context.Background(), *perfqueryTimeout)
  defer cancelExtended()
  ports := getDevicePorts(device.Uplinks)
@@ -308,7 +316,6 @@ func (h *HCACollector) collect() ([]PerfQueryCounters, float64, float64) {
  errors++
  }
  if err != nil {
- <-limit
  return
  }
  deviceCounters, errs := perfqueryParse(device, extendedOut, h.logger)
@@ -340,7 +347,6 @@ func (h *HCACollector) collect() ([]PerfQueryCounters, float64, float64) {
  countersLock.Unlock()
  }
  }
- <-limit
  }(device)
  }
  wg.Wait()

diff --git a/collectors/hca_test.go b/collectors/hca_test.go
@@ -24,12 +24,12 @@ import (
 
 var (
  hcaDevices = []InfinibandDevice{
- {Type: "CA", LID: "133", GUID: "0x7cfe9003003b4b96", Rate: (25 * 4 * 125000000), Name: "o0002",
+ {Type: "CA", LID: "133", GUID: "0x7cfe9003003b4b96", Rate: (25 * 4 * 125000000), RawRate: 1.2890625e+10, Name: "o0002",
  Uplinks: map[string]InfinibandUplink{
  "1": {Type: "SW", LID: "1719", PortNumber: "11", GUID: "0x7cfe9003009ce5b0", Name: "ib-i1l1s01"},
  },
  },
- {Type: "CA", LID: "134", GUID: "0x7cfe9003003b4bde", Rate: (25 * 4 * 125000000), Name: "o0001",
+ {Type: "CA", LID: "134", GUID: "0x7cfe9003003b4bde", Rate: (25 * 4 * 125000000), RawRate: 1.2890625e+10, Name: "o0001",
  Uplinks: map[string]InfinibandUplink{
  "1": {Type: "SW", LID: "1719", PortNumber: "10", GUID: "0x7cfe9003009ce5b0", Name: "ib-i1l1s01"},
  },
@@ -145,6 +145,10 @@ func TestHCACollector(t *testing.T) {
  # TYPE infiniband_hca_rate_bytes_per_second gauge
  infiniband_hca_rate_bytes_per_second{guid="0x7cfe9003003b4b96"} 1.25e+10
  infiniband_hca_rate_bytes_per_second{guid="0x7cfe9003003b4bde"} 1.25e+10
+ # HELP infiniband_hca_raw_rate_bytes_per_second Infiniband HCA raw rate
+ # TYPE infiniband_hca_raw_rate_bytes_per_second gauge
+ infiniband_hca_raw_rate_bytes_per_second{guid="0x7cfe9003003b4b96"} 1.2890625e+10
+ infiniband_hca_raw_rate_bytes_per_second{guid="0x7cfe9003003b4bde"} 1.2890625e+10
  # HELP infiniband_hca_uplink_info Infiniband HCA uplink information
  # TYPE infiniband_hca_uplink_info gauge
  infiniband_hca_uplink_info{guid="0x7cfe9003003b4b96",hca="o0002",port="1",uplink="ib-i1l1s01",uplink_guid="0x7cfe9003009ce5b0",uplink_lid="1719",uplink_port="11",uplink_type="SW"} 1
@@ -154,8 +158,8 @@ func TestHCACollector(t *testing.T) {
  gatherers := setupGatherer(collector)
  if val, err := testutil.GatherAndCount(gatherers); err != nil {
  t.Errorf("Unexpected error: %v", err)
- } else if val != 53 {
- t.Errorf("Unexpected collection count %d, expected 53", val)
+ } else if val != 55 {
+ t.Errorf("Unexpected collection count %d, expected 55", val)
  }
  if err := testutil.GatherAndCompare(gatherers, strings.NewReader(expected),
  "infiniband_hca_port_excessive_buffer_overrun_errors_total", "infiniband_hca_port_link_downed_total",
@@ -170,7 +174,7 @@ func TestHCACollector(t *testing.T) {
  "infiniband_hca_port_transmit_wait_total", "infiniband_hca_port_unicast_receive_packets_total",
  "infiniband_hca_port_unicast_transmit_packets_total", "infiniband_hca_port_vl15_dropped_total",
  "infiniband_hca_port_buffer_overrun_errors_total",
- "infiniband_hca_info", "infiniband_hca_rate_bytes_per_second", "infiniband_hca_uplink_info",
+ "infiniband_hca_info", "infiniband_hca_rate_bytes_per_second", "infiniband_hca_raw_rate_bytes_per_second", "infiniband_hca_uplink_info",
  "infiniband_exporter_collect_errors", "infiniband_exporter_collect_timeouts"); err != nil {
  t.Errorf("unexpected collecting result:\n%s", err)
  }
@@ -308,6 +312,10 @@ func TestHCACollectorFull(t *testing.T) {
  # TYPE infiniband_hca_rate_bytes_per_second gauge
  infiniband_hca_rate_bytes_per_second{guid="0x7cfe9003003b4b96"} 1.25e+10
  infiniband_hca_rate_bytes_per_second{guid="0x7cfe9003003b4bde"} 1.25e+10
+ # HELP infiniband_hca_raw_rate_bytes_per_second Infiniband HCA raw rate
+ # TYPE infiniband_hca_raw_rate_bytes_per_second gauge
+ infiniband_hca_raw_rate_bytes_per_second{guid="0x7cfe9003003b4b96"} 1.2890625e+10
+ infiniband_hca_raw_rate_bytes_per_second{guid="0x7cfe9003003b4bde"} 1.2890625e+10
  # HELP infiniband_hca_uplink_info Infiniband HCA uplink information
  # TYPE infiniband_hca_uplink_info gauge
  infiniband_hca_uplink_info{guid="0x7cfe9003003b4b96",hca="o0002",port="1",uplink="ib-i1l1s01",uplink_guid="0x7cfe9003009ce5b0",uplink_lid="1719",uplink_port="11",uplink_type="SW"} 1
@@ -317,8 +325,8 @@ func TestHCACollectorFull(t *testing.T) {
  gatherers := setupGatherer(collector)
  if val, err := testutil.GatherAndCount(gatherers); err != nil {
  t.Errorf("Unexpected error: %v", err)
- } else if val != 65 {
- t.Errorf("Unexpected collection count %d, expected 65", val)
+ } else if val != 67 {
+ t.Errorf("Unexpected collection count %d, expected 67", val)
  }
  if err := testutil.GatherAndCompare(gatherers, strings.NewReader(expected),
  "infiniband_hca_port_excessive_buffer_overrun_errors_total", "infiniband_hca_port_link_downed_total",
@@ -335,7 +343,7 @@ func TestHCACollectorFull(t *testing.T) {
  "infiniband_hca_port_buffer_overrun_errors_total", "infiniband_hca_port_dli_mapping_errors_total",
  "infiniband_hca_port_local_physical_errors_total", "infiniband_hca_port_looping_errors_total",
  "infiniband_hca_port_malformed_packet_errors_total", "infiniband_hca_port_vl_mapping_errors_total",
- "infiniband_hca_info", "infiniband_hca_rate_bytes_per_second", "infiniband_hca_uplink_info",
+ "infiniband_hca_info", "infiniband_hca_rate_bytes_per_second", "infiniband_hca_raw_rate_bytes_per_second", "infiniband_hca_uplink_info",
  "infiniband_exporter_collect_errors", "infiniband_exporter_collect_timeouts"); err != nil {
  t.Errorf("unexpected collecting result:\n%s", err)
  }
@@ -358,8 +366,8 @@ func TestHCACollectorError(t *testing.T) {
  gatherers := setupGatherer(collector)
  if val, err := testutil.GatherAndCount(gatherers); err != nil {
  t.Errorf("Unexpected error: %v", err)
- } else if val != 9 {
- t.Errorf("Unexpected collection count %d, expected 9", val)
+ } else if val != 11 {
+ t.Errorf("Unexpected collection count %d, expected 11", val)
  }
  if err := testutil.GatherAndCompare(gatherers, strings.NewReader(expected),
  "infiniband_hca_port_excessive_buffer_overrun_errors_total", "infiniband_hca_port_link_downed_total",
@@ -386,8 +394,8 @@ func TestHCACollectorErrorRunonce(t *testing.T) {
  gatherers := setupGatherer(collector)
  if val, err := testutil.GatherAndCount(gatherers); err != nil {
  t.Errorf("Unexpected error: %v", err)
- } else if val != 10 {
- t.Errorf("Unexpected collection count %d, expected 10", val)
+ } else if val != 12 {
+ t.Errorf("Unexpected collection count %d, expected 12", val)
  }
  if err := testutil.GatherAndCompare(gatherers, strings.NewReader(expected),
  "infiniband_hca_port_excessive_buffer_overrun_errors_total", "infiniband_hca_port_link_downed_total",
@@ -414,8 +422,8 @@ func TestHCACollectorTimeout(t *testing.T) {
  gatherers := setupGatherer(collector)
  if val, err := testutil.GatherAndCount(gatherers); err != nil {
  t.Errorf("Unexpected error: %v", err)
- } else if val != 9 {
- t.Errorf("Unexpected collection count %d, expected 9", val)
+ } else if val != 11 {
+ t.Errorf("Unexpected collection count %d, expected 11", val)
  }
  if err := testutil.GatherAndCompare(gatherers, strings.NewReader(expected),
  "infiniband_hca_port_excessive_buffer_overrun_errors_total", "infiniband_hca_port_link_downed_total",

diff --git a/collectors/ibnetdiscover.go b/collectors/ibnetdiscover.go
@@ -35,16 +35,18 @@ var (
  ibnetdiscoverPath = kingpin.Flag("ibnetdiscover.path", "Path to ibnetdiscover").Default("ibnetdiscover").String()
  nodeNameMap = kingpin.Flag("ibnetdiscover.node-name-map", "Path to node name map file").Default("").String()
  ibnetdiscoverTimeout = kingpin.Flag("ibnetdiscover.timeout", "Timeout for ibnetdiscover execution").Default("20s").Duration()
- rates = map[string]float64{
- "SDR": 2,
- "DDR": 4,
- "QDR": 8,
- "FDR10": 10,
- "FDR": 14,
- "EDR": 25,
- "HDR": 50,
- "NDR": 100,
- "XDR": 250,
+ // IB Lane Rate Specification: {signaling rate, effective rate}, Gbps
+ // https://en.wikipedia.org/wiki/InfiniBand#Performance
+ laneRates = map[string][]float64{
+ "SDR": {2.5, 2},
+ "DDR": {5, 4},
+ "QDR": {10, 8},
+ "FDR10": {10.3125, 10},
+ "FDR": {14.0625, 13.64},
+ "EDR": {25.78125, 25},
+ "HDR": {50, 50},
+ "NDR": {100, 100},
+ "XDR": {250, 250},
  }
 )
 
@@ -53,6 +55,7 @@ type InfinibandDevice struct {
  LID string
  GUID string
  Rate float64
+ RawRate float64
  Name string
  Uplinks map[string]InfinibandUplink
 }
@@ -135,6 +138,17 @@ func ibnetdiscoverParse(out string, logger log.Logger) (*[]InfinibandDevice, *[]
  level.Debug(logger).Log("msg", "Skipping line that is not connected", "line", line)
  continue
  }
+ // check the last item, because name may have space so that it is split into multiple items
+ name := items[len(items)-1]
+ if strings.HasSuffix(name, `'`) && !isPairedQuotesName(name) {
+ for i := len(items) - 2; i > 5; i-- {
+ name = items[i] + name
+ if isPairedQuotesName(name) {
+ items = append(items[:i], name)
+ break
+ }
+ }
+ }
  if items[5] == "SDR" && len(items) == 7 {
  level.Debug(logger).Log("msg", "Skipping split mode port", "line", line)
  continue
@@ -151,12 +165,13 @@ func ibnetdiscoverParse(out string, logger log.Logger) (*[]InfinibandDevice, *[]
  device.Type = items[0]
  device.LID = items[1]
  device.GUID = guid
- rate, err := parseRate(items[4], items[5])
+ rawRate, effectiveRate, err := parseRate(items[4], items[5])
  if err != nil {
  level.Error(logger).Log("msg", "Unable to parse speed", "width", items[4], "rate", items[5], "type", device.Type, "guid", device.GUID)
  return nil, nil, err
  } else {
- device.Rate = rate
+ device.Rate = effectiveRate
+ device.RawRate = rawRate
  }
  portName, uplinkName, err := parseNames(line)
  if err != nil {
@@ -186,20 +201,28 @@ func ibnetdiscoverParse(out string, logger log.Logger) (*[]InfinibandDevice, *[]
  return &switches, &hcas, nil
 }
 
-func parseRate(width string, rateStr string) (float64, error) {
- var rate float64
+func parseRate(width string, rateStr string) (float64, float64, error) {
  widthRe := regexp.MustCompile("[0-9]+")
  widthMatch := widthRe.FindAllString(width, 1)
  if len(widthMatch) != 1 {
- return 0, fmt.Errorf("Unable to find match for %s: %v", width, widthMatch)
+ return 0, 0, fmt.Errorf("Unable to find match for %s: %v", width, widthMatch)
  }
  widthMultipler, _ := strconv.ParseFloat(widthMatch[0], 64)
- if baseRate, ok := rates[rateStr]; ok {
- rate = widthMultipler * baseRate * math.Pow(1000, 3) / 8
- } else {
- return 0, fmt.Errorf("Unknown rate %s", rateStr)
+ if laneRate, ok := laneRates[rateStr]; ok {
+ baseRate := widthMultipler * math.Pow(1000, 3) / 8
+ rawRate := laneRate[0] * baseRate
+ effectiveRate := laneRate[1] * baseRate
+ return rawRate, effectiveRate, nil
+ }
+ return 0, 0, fmt.Errorf("Unknown rate %s", rateStr)
+}
+
+func isPairedQuotesName(name string) bool {
+ if name == `'` {
+ return false
  }
- return rate, nil
+ first, last := name[0], name[len(name)-1]
+ return first == last && first == '\''
 }
 
 func parseNames(line string) (string, string, error) {
@@ -208,8 +231,8 @@ func parseNames(line string) (string, string, error) {
  if len(matches) != 3 {
  return "", "", fmt.Errorf("Unable to extract names using regexp")
  }
- portName := matches[1]
- uplinkName := matches[2]
+ portName := strings.TrimSpace(matches[1])
+ uplinkName := strings.TrimSpace(matches[2])
  if strings.Contains(portName, " HCA") {
  portName = strings.Split(portName, " ")[0]
  }