Merge pull request #8 from treydock/ssh_output

Add opt-in `ssh_output` metric

CHANGELOG.md

@@ -1,3 +1,9 @@
+## Unreleased
+
+### Changes
+
+* [ENHANCEMENT] Add opt-in `ssh_output` metric
+
 ## 1.0.0 / 2020-11-22
 
 * Update to 1.15 and update Go module dependencies

README.md

@@ -42,6 +42,12 @@ modules:
  command: uptime
  command_expect: "load average"
  timeout: 5
+ capture:
+ user: prometheus
+ private_key: /home/prometheus/.ssh/id_rsa
+ command: /some/command/with/output
+ output_metric: true
+ output_truncate: 50
 ```
 
 Example with curl would query host1 with the password module and host2 with the default module.
@@ -63,6 +69,8 @@ Configuration options for each module:
  * The default comes from the `--collector.ssh.default-timeout` flag.
 * `command` - Optional command to run.
 * `command_expect` - Optional regular expression of output to expect from the command.
+* `output_metric` - If `true` the exporter will expose the `command` output via `ssh_output{output="<output here>"}` metric.
+* `output_truncate` - Sets the max length for a string in `ssh_output` metric's `output` label. Set to `-1` to disable truncating.
 
 ## Docker
 

collector/collector.go

@@ -37,12 +37,14 @@ const (
 type Metric struct {
  Success float64
  FailureReason string
+ Output string
 }
 
 type Collector struct {
  Success *prometheus.Desc
  Duration *prometheus.Desc
  Failure *prometheus.Desc
+ Output *prometheus.Desc
  target *config.Target
  logger log.Logger
 }
@@ -55,6 +57,8 @@ func NewCollector(target *config.Target, logger log.Logger) *Collector {
  "How long the SSH check took in seconds", nil, nil),
  Failure: prometheus.NewDesc(prometheus.BuildFQName(namespace, "", "failure"),
  "Indicates a failure", []string{"reason"}, nil),
+ Output: prometheus.NewDesc(prometheus.BuildFQName(namespace, "", "output"),
+ "The output of the executed command", []string{"output"}, nil),
  target: target,
  logger: logger,
  }
@@ -64,6 +68,7 @@ func (c *Collector) Describe(ch chan<- *prometheus.Desc) {
  ch <- c.Success
  ch <- c.Duration
  ch <- c.Failure
+ ch <- c.Output
 }
 
 func (c *Collector) Collect(ch chan<- prometheus.Metric) {
@@ -81,6 +86,10 @@ func (c *Collector) Collect(ch chan<- prometheus.Metric) {
  }
  ch <- prometheus.MustNewConstMetric(c.Failure, prometheus.GaugeValue, value, reason)
  }
+ if c.target.OutputMetric {
+ output := truncateString(metric.Output, c.target.OutputTruncate)
+ ch <- prometheus.MustNewConstMetric(c.Output, prometheus.GaugeValue, 1, strings.TrimSuffix(output, "\n"))
+ }
  ch <- prometheus.MustNewConstMetric(c.Duration, prometheus.GaugeValue, time.Since(collectTime).Seconds())
 }
 
@@ -90,7 +99,6 @@ func (c *Collector) collect() Metric {
  var metric Metric
  var auth ssh.AuthMethod
  var sessionerror, autherror, commanderror error
- var commandOutput string
 
  if c.target.PrivateKey != "" {
  auth, autherror = getPrivateKeyAuth(c.target.PrivateKey)
@@ -136,7 +144,7 @@ func (c *Collector) collect() Metric {
  if commanderror != nil {
  return
  }
- commandOutput = cmdBuffer.String()
+ metric.Output = cmdBuffer.String()
  }
  if !timeout {
  c1 <- 1
@@ -165,9 +173,9 @@ func (c *Collector) collect() Metric {
  }
  if c.target.Command != "" && c.target.CommandExpect != "" {
  commandExpectPattern := regexp.MustCompile(c.target.CommandExpect)
- if !commandExpectPattern.MatchString(commandOutput) {
+ if !commandExpectPattern.MatchString(metric.Output) {
  level.Error(c.logger).Log("msg", "Command output did not match expected value",
- "output", commandOutput, "command", c.target.Command)
+ "output", metric.Output, "command", c.target.Command)
  metric.FailureReason = "command-output"
  return metric
  }
@@ -207,3 +215,17 @@ func hostKeyCallback(metric *Metric, target *config.Target, logger log.Logger) s
  return hostKeyCallback(hostname, remote, key)
  }
 }
+
+func truncateString(str string, num int) string {
+ bnoden := str
+ if num == -1 {
+ return bnoden
+ }
+ if len(str) > num {
+ if num > 3 {
+ num -= 3
+ }
+ bnoden = str[0:num] + "..."
+ }
+ return bnoden
+}

collector/collector_test.go

@@ -136,7 +136,7 @@ func TestCollector(t *testing.T) {
  t.Errorf("Unexpected collection count %d, expected 6", val)
  }
  if err := testutil.GatherAndCompare(gatherers, strings.NewReader(expected),
- "ssh_success", "ssh_failure"); err != nil {
+ "ssh_success", "ssh_failure", "ssh_output"); err != nil {
  t.Errorf("unexpected collecting result:\n%s", err)
  }
 }
@@ -171,7 +171,87 @@ func TestCollectorCommand(t *testing.T) {
  t.Errorf("Unexpected collection count %d, expected 6", val)
  }
  if err := testutil.GatherAndCompare(gatherers, strings.NewReader(expected),
- "ssh_success", "ssh_failure"); err != nil {
+ "ssh_success", "ssh_failure", "ssh_output"); err != nil {
+ t.Errorf("unexpected collecting result:\n%s", err)
+ }
+}
+
+func TestCollectorCommandOutputMetric(t *testing.T) {
+ expected := `
+ # HELP ssh_failure Indicates a failure
+ # TYPE ssh_failure gauge
+ ssh_failure{reason="command-error"} 0
+ ssh_failure{reason="command-output"} 0
+ ssh_failure{reason="error"} 0
+ ssh_failure{reason="timeout"} 0
+ # HELP ssh_output The output of the executed command
+ # TYPE ssh_output gauge
+ ssh_output{output="11:42:20 up 57 days, 19:18, 5 users, load ave..."} 1
+ # HELP ssh_success SSH connection was successful
+ # TYPE ssh_success gauge
+ ssh_success 1
+ `
+ target := &config.Target{
+ Host: fmt.Sprintf("localhost:%d", listen),
+ User: "test",
+ Password: "test",
+ Command: "uptime",
+ CommandExpect: "load average",
+ OutputMetric: true,
+ OutputTruncate: 50,
+ Timeout: 2,
+ }
+ w := log.NewSyncWriter(os.Stderr)
+ logger := log.NewLogfmtLogger(w)
+ collector := NewCollector(target, logger)
+ gatherers := setupGatherer(collector)
+ if val, err := testutil.GatherAndCount(gatherers); err != nil {
+ t.Errorf("Unexpected error: %v", err)
+ } else if val != 7 {
+ t.Errorf("Unexpected collection count %d, expected 7", val)
+ }
+ if err := testutil.GatherAndCompare(gatherers, strings.NewReader(expected),
+ "ssh_success", "ssh_failure", "ssh_output"); err != nil {
+ t.Errorf("unexpected collecting result:\n%s", err)
+ }
+}
+
+func TestCollectorCommandOutputMetricNoTruncate(t *testing.T) {
+ expected := `
+ # HELP ssh_failure Indicates a failure
+ # TYPE ssh_failure gauge
+ ssh_failure{reason="command-error"} 0
+ ssh_failure{reason="command-output"} 0
+ ssh_failure{reason="error"} 0
+ ssh_failure{reason="timeout"} 0
+ # HELP ssh_output The output of the executed command
+ # TYPE ssh_output gauge
+ ssh_output{output="11:42:20 up 57 days, 19:18, 5 users, load average: 2.48, 1.10, 0.49"} 1
+ # HELP ssh_success SSH connection was successful
+ # TYPE ssh_success gauge
+ ssh_success 1
+ `
+ target := &config.Target{
+ Host: fmt.Sprintf("localhost:%d", listen),
+ User: "test",
+ Password: "test",
+ Command: "uptime",
+ CommandExpect: "load average",
+ OutputMetric: true,
+ OutputTruncate: -1,
+ Timeout: 2,
+ }
+ w := log.NewSyncWriter(os.Stderr)
+ logger := log.NewLogfmtLogger(w)
+ collector := NewCollector(target, logger)
+ gatherers := setupGatherer(collector)
+ if val, err := testutil.GatherAndCount(gatherers); err != nil {
+ t.Errorf("Unexpected error: %v", err)
+ } else if val != 7 {
+ t.Errorf("Unexpected collection count %d, expected 7", val)
+ }
+ if err := testutil.GatherAndCompare(gatherers, strings.NewReader(expected),
+ "ssh_success", "ssh_failure", "ssh_output"); err != nil {
  t.Errorf("unexpected collecting result:\n%s", err)
  }
 }
@@ -204,7 +284,7 @@ func TestCollectorCommandOutputError(t *testing.T) {
  t.Errorf("Unexpected collection count %d, expected 6", val)
  }
  if err := testutil.GatherAndCompare(gatherers, strings.NewReader(expected),
- "ssh_success", "ssh_failure"); err != nil {
+ "ssh_success", "ssh_failure", "ssh_output"); err != nil {
  t.Errorf("unexpected collecting result:\n%s", err)
  }
 }
@@ -237,7 +317,7 @@ func TestCollectorTimeoutDial(t *testing.T) {
  t.Errorf("Unexpected collection count %d, expected 6", val)
  }
  if err := testutil.GatherAndCompare(gatherers, strings.NewReader(expected),
- "ssh_success", "ssh_failure"); err != nil {
+ "ssh_success", "ssh_failure", "ssh_output"); err != nil {
  t.Errorf("unexpected collecting result:\n%s", err)
  }
 }
@@ -271,7 +351,7 @@ func TestCollectorTimeoutCommand(t *testing.T) {
  t.Errorf("Unexpected collection count %d, expected 6", val)
  }
  if err := testutil.GatherAndCompare(gatherers, strings.NewReader(expected),
- "ssh_success", "ssh_failure"); err != nil {
+ "ssh_success", "ssh_failure", "ssh_output"); err != nil {
  t.Errorf("unexpected collecting result:\n%s", err)
  }
 }
@@ -304,7 +384,7 @@ func TestCollectorError(t *testing.T) {
  t.Errorf("Unexpected collection count %d, expected 6", val)
  }
  if err := testutil.GatherAndCompare(gatherers, strings.NewReader(expected),
- "ssh_success", "ssh_failure"); err != nil {
+ "ssh_success", "ssh_failure", "ssh_output"); err != nil {
  t.Errorf("unexpected collecting result:\n%s", err)
  }
 }
@@ -337,7 +417,7 @@ func TestCollectorPrivateKey(t *testing.T) {
  t.Errorf("Unexpected collection count %d, expected 6", val)
  }
  if err := testutil.GatherAndCompare(gatherers, strings.NewReader(expected),
- "ssh_success", "ssh_failure"); err != nil {
+ "ssh_success", "ssh_failure", "ssh_output"); err != nil {
  t.Errorf("unexpected collecting result:\n%s", err)
  }
 }
@@ -371,7 +451,7 @@ func TestCollectorKnownHosts(t *testing.T) {
  t.Errorf("Unexpected collection count %d, expected 6", val)
  }
  if err := testutil.GatherAndCompare(gatherers, strings.NewReader(expected),
- "ssh_success", "ssh_failure"); err != nil {
+ "ssh_success", "ssh_failure", "ssh_output"); err != nil {
  t.Errorf("unexpected collecting result:\n%s", err)
  }
 }
@@ -405,7 +485,7 @@ func TestCollectorKnownHostsError(t *testing.T) {
  t.Errorf("Unexpected collection count %d, expected 6", val)
  }
  if err := testutil.GatherAndCompare(gatherers, strings.NewReader(expected),
- "ssh_success", "ssh_failure"); err != nil {
+ "ssh_success", "ssh_failure", "ssh_output"); err != nil {
  t.Errorf("unexpected collecting result:\n%s", err)
  }
 }
@@ -439,7 +519,7 @@ func TestCollectorKnownHostsDNE(t *testing.T) {
  t.Errorf("Unexpected collection count %d, expected 6", val)
  }
  if err := testutil.GatherAndCompare(gatherers, strings.NewReader(expected),
- "ssh_success", "ssh_failure"); err != nil {
+ "ssh_success", "ssh_failure", "ssh_output"); err != nil {
  t.Errorf("unexpected collecting result:\n%s", err)
  }
 }

config/config.go

@@ -23,7 +23,9 @@ import (
 )
 
 var (
- defaultTimeout = kingpin.Flag("collector.ssh.default-timeout", "Default timeout for SSH collection").Default("10").Int()
+ defaultTimeout = kingpin.Flag("collector.ssh.default-timeout", "Default timeout for SSH collection").Default("10").Int()
+ defaultOutputTruncate = kingpin.Flag("collector.ssh.default-output-truncate",
+ "Default output truncate length when output metric is enabled").Default("50").Int()
 )
 
 type Config struct {
@@ -45,6 +47,8 @@ type Module struct {
  Timeout int `yaml:"timeout"`
  Command string `yaml:"command"`
  CommandExpect string `yaml:"command_expect"`
+ OutputMetric bool `yaml:"output_metric"`
+ OutputTruncate int `yaml:"output_truncate"`
 }
 
 type Target struct {
@@ -57,6 +61,8 @@ type Target struct {
  Timeout int
  Command string
  CommandExpect string
+ OutputMetric bool
+ OutputTruncate int
 }
 
 func (sc *SafeConfig) ReloadConfig(configFile string) error {
@@ -83,6 +89,9 @@ func (sc *SafeConfig) ReloadConfig(configFile string) error {
  if module.Timeout == 0 {
  module.Timeout = *defaultTimeout
  }
+ if module.OutputTruncate == 0 {
+ module.OutputTruncate = *defaultOutputTruncate
+ }
  c.Modules[key] = module
  }
  sc.Lock()

config/config_test.go

@@ -41,6 +41,9 @@ func TestReloadConfigDefaults(t *testing.T) {
  if module.Timeout != 10 {
  t.Errorf("Module Timeout does not match default 10")
  }
+ if module.OutputTruncate != 50 {
+ t.Errorf("Module OutputTruncate does not match default 50")
+ }
 }
 
 func TestReloadConfigBadConfigs(t *testing.T) {