Upgrade Dependencies and Added new Metrics

This release is an overdue maintenance update, courtesy of @talatuyarer. (#13)

Add --include-jobs-regex option

Add --include-jobs-regex to allow filtering out unwanted jobs

Allow overriding KAFKA_GROUP_ID via environment variable

Allow overriding KAFKA_GROUP_ID via environment variable

Add more metrics

Add metrics from org.apache.samza.system.chooser.BootstrappingChooserMetrics, courtesy of @juris.

Don't crash on unrecognized metric

The exporter now prints a warning to the console instead of crashing on an uncrecognized metric.

Misc Bugfixes

• Add host to metric labels
• Fix missing container-name in labels
• Fix crash on Kafka systems not named kafka

Add TTL to Gauges and Gauge label sets

The exporter now stops reporting metrics after no update has been seen for 60s (default).
Also, if a Gauge labelset has not been updated for 60s, that labelset is no longer reported.

Add samza_task_name label to all metrics.

Also fix the package name to samza_prometheus_exporter.

Add more Samza 0.10 metrics

0.1.3

Add new metrics for JvmMetrics

Initial Implementation

• Supports the Samza metrics listed below.
• Unrecognized metric classes only get the samza_job label.

{
    'org.apache.samza.system.kafka.KafkaSystemProducerMetrics': [
        'kafka-producer-send-failed',
        'kafka-producer-send-success',
        'kafka-producer-sends',
        'kafka-producer-retries',
        'kafka-flush-ms',
        'kafka-flush-failed',
        'kafka-flushes',
        'serialization error',
    ],
    'org.apache.samza.system.kafka.KafkaSystemConsumerMetrics': {
        (re('(.*)-(\d+)-(bytes-read)'), topic_partition_metric),
        (re('(.*)-(\d+)-(high-watermark)'), topic_partition_metric),
        (re('(.*)-(\d+)-(messages-read)'), topic_partition_metric),
        (re('(.*)-(\d+)-(offset-change)'), topic_partition_metric),
        (re('(.*)-(\d+)-(messages-behind-high-watermark)'), topic_partition_metric),
        (re('(.*)-(reconnects)'), system_metric),
        (re('(.*)-(skipped-fetch-requests)'), system_metric),
        (re('(.*)-(topic-partitions)'), system_metric),
        (re('(.*)-SystemStreamPartition \[(.*), (.*), (.*)\]'), kafka_system_stream_partition_metric),
        'poll-count',
    },
    'org.apache.samza.system.SystemProducersMetrics': {
        'flushes',
        'sends',
        'serialization error',
        (re('(.*)-(flushes)'), source_metric),
        (re('(.*)-(sends)'), source_metric),
    },
    'org.apache.samza.system.SystemConsumersMetrics': {
        'chose-null',
        'chose-object',
        'unprocessed-messages',
        'poll-timeout',
        'ssps-needed-by-chooser',
        'deserialization error',
        'kafka-messages-per-poll',
        'kafka-polls',
        'kafka-ssp-fetches-per-poll',
        (re('([^-]*)-(.*)-(messages-chosen)'), system_topic_metric),
    },
    'org.apache.samza.metrics.JvmMetrics': {
        'mem-heap-committed-mb',
        'mem-heap-used-mb',
        'mem-non-heap-committed-mb',
        'mem-non-heap-used-mb',
        'threads-blocked',
        'threads-terminated',
        'threads-timed-waiting',
        'threads-runnable',
        'threads-waiting',
        'threads-new',
        'thread new',
        'ps marksweep-gc-count',
        'ps marksweep-gc-time-millis',
        'ps scavenge-gc-count',
        'ps scavenge-gc-time-millis',
        'gc-count',
        'gc-time-millis',
    },
    'org.apache.samza.job.yarn.SamzaAppMasterMetrics': {
        'released-containers',
        'completed-containers',
        'needed-containers',
        'running-containers',
        'failed-containers',
        'released-containers',
        'job-healthy',
        'http-port',
        'rpc-port',
        'task-count',
    },
    'org.apache.samza.container.SamzaContainerMetrics': {
        'process-calls',
        'window-calls',
        'send-calls',
        'commit-calls',
        'process-null-envelopes',
        'process-envelopes',
        'process-ms',
        'window-ms',
        'choose-ms',
        'commit-ms',
    },
    'org.apache.samza.storage.kv.KeyValueStoreMetrics': {
        (re('(.*)-(bytes-read)'), store_metric),
        (re('(.*)-(bytes-written)'), store_metric),
        (re('(.*)-(flushes)'), store_metric),
        (re('(.*)-(deletes)'), store_metric),
        (re('(.*)-(puts)'), store_metric),
        (re('(.*)-(alls)'), store_metric),
        (re('(.*)-(ranges)'), store_metric),
        (re('(.*)-(gets)'), store_metric),
    },
    'org.apache.samza.storage.kv.LoggedStoreMetrics': {
        (re('(.*)-(gets)'), store_metric),
        (re('(.*)-(puts)'), store_metric),
        (re('(.*)-(alls)'), store_metric),
        (re('(.*)-(deletes)'), store_metric),
        (re('(.*)-(ranges)'), store_metric),
        (re('(.*)-(flushes)'), store_metric),
    },
    'org.apache.samza.storage.kv.SerializedKeyValueStoreMetrics': {
        (re('(.*)-(flushes)'), store_metric),
        (re('(.*)-(ranges)'), store_metric),
        (re('(.*)-(deletes)'), store_metric),
        (re('(.*)-(puts)'), store_metric),
        (re('(.*)-(gets)'), store_metric),
        (re('(.*)-(bytes-deserialized)'), store_metric),
        (re('(.*)-(bytes-serialized)'), store_metric),
        (re('(.*)-(alls)'), store_metric),
    },
    'org.apache.samza.storage.kv.KeyValueStorageEngineMetrics': {
        (re('(.*)-(ranges)'), store_metric),
        (re('(.*)-(flushes)'), store_metric),
        (re('(.*)-(puts)'), store_metric),
        (re('(.*)-(bytes-written)'), store_metric),
        (re('(.*)-(gets)'), store_metric),
        (re('(.*)-(messages-bytes)'), store_metric),
        (re('(.*)-(alls)'), store_metric),
        (re('(.*)-(messages-restored)'), store_metric),
        (re('(.*)-(deletes)'), store_metric),
        (re('(.*)-(bytes-deserialized)'), store_metric),
    },
    'org.apache.samza.container.TaskInstanceMetrics': {
        'commit-calls',
        'window-calls',
        'process-calls',
        'flush-calls',
        'send-calls',
        'messages-sent',
        (re('([^-]*)-(.*)-(\d+)-(offset)'), system_stream_partition_metric),
    },
    'org.apache.samza.storage.kv.CachedStoreMetrics': {
        (re('(.*)-(cache-hits)'), store_metric),
        (re('(.*)-(flushes)'), store_metric),
        (re('(.*)-(store-calls)'), store_metric),
        (re('(.*)-(messages-restored)'), store_metric),
        (re('(.*)-(flush-batch-size)'), store_metric),
        (re('(.*)-(alls)'), store_metric),
        (re('(.*)-(dirty-count)'), store_metric),
        (re('(.*)-(puts)'), store_metric),
        (re('(.*)-(gets)'), store_metric),
        (re('(.*)-(deletes)'), store_metric),
        (re('(.*)-(ranges)'), store_metric),
        (re('(.*)-(cache-size)'), store_metric),
    },
    'org.apache.samza.system.chooser.RoundRobinChooserMetrics': {
        'buffered-messages',
    },
}