parent
1004645e0a
commit
64766d9e14
31 changed files with 1638 additions and 2 deletions
@ -0,0 +1,13 @@ |
||||
# Changelog |
||||
|
||||
All notable changes to this project will be documented in this file. |
||||
|
||||
Added : for new features. |
||||
Changed : for changes in existing functionality. |
||||
Deprecated: for soon-to-be removed features. |
||||
Removed : for now removed features. |
||||
Fixed : for any bug fixes. |
||||
Security : in case of vulnerabilities. |
||||
|
||||
## [1.0.0] - 2022-10-28 |
||||
First working version. |
||||
@ -1,3 +1,56 @@ |
||||
# victron_mqtt_exporter |
||||
# Prometheus exporter for Victron MQTT devices |
||||
|
||||
victron_mqtt_exporter |
||||
A purposely build Prometheus exporter for Victron Energy Cerbo GX MQTT devices. |
||||
|
||||
(C) 2022 M. Konstapel https://meezenest.nl/mees |
||||
|
||||
Based on the General purpose Prometheus exporter for MQTT from Frederic Hemberger. |
||||
(https://github.com/fhemberger/mqtt_exporter) |
||||
|
||||
Subscribes to one or more MQTT topics, and lets you configure prometheus metrics based on pattern matching. |
||||
|
||||
## Features |
||||
|
||||
- Converts JSON string from Victron MQTT device topics to value readable by Prometheus. |
||||
- Sends keepalive signal to Victron Energy Cerbo GX MQTT device (every 30 seconds). |
||||
- Supported Metrics: |
||||
- standard metrics |
||||
- Gauge, Counter, Histogram, Summary |
||||
- additional |
||||
- **Counter (Absolute):** |
||||
- Same as Counter, but working with absolute numbers received from MQTT. Which is far more common, than sending the diff in each publish. |
||||
- e.g. a network counter or a rain sensor |
||||
- **Enum:** |
||||
- is a metric type not so common, details can be found in the [OpenMetrics docs](https://github.com/OpenObservability/OpenMetrics/blob/main/specification/OpenMetrics.md#stateset) and [Python client code](https://github.com/prometheus/client_python/blob/9a24236695c9ad47f9dc537a922a6d1333d8d093/prometheus_client/metrics.py#L640-L698). |
||||
- Allows to track as state by a know set of strings describing the state, e.g. `on/off` or `high/medium/low` |
||||
- Common sources would be a light switch oder a door lock. |
||||
- Comprehensive rewriting for topic, value/payload and labels |
||||
- similar to prometheus label rewrites |
||||
- regex allows almost every conversion |
||||
- e.g. to |
||||
- remove units or other strings from payload |
||||
- convert topic hierarchy into labels |
||||
- normalize labels |
||||
- check example configs `./exampleconf` and the configs in `./test/test_data/` |
||||
|
||||
|
||||
## Usage |
||||
|
||||
- Create a folder to hold the config (default: `conf/`) |
||||
- Add metric config(s) in YAML format to the folder. Files are combined and read as a single config. (See `exampleconf/metric_example.yaml` for details) |
||||
- Install dependencies with `pip3 install -r requirements-frozen.txt` |
||||
- Run `./victron_mqtt_exporter.py` |
||||
|
||||
|
||||
## Python dependencies |
||||
|
||||
- paho-mqtt |
||||
- prometheus-client |
||||
- PyYAML |
||||
- yamlreader |
||||
|
||||
|
||||
## TODO |
||||
|
||||
- Add persistence of metrics on restart |
||||
- forget/age out metrics receiving no updates anymore |
||||
|
||||
@ -0,0 +1,26 @@ |
||||
# Config file for MQTT prometheus exporter |
||||
|
||||
# Logging |
||||
#logging: |
||||
# logfile: '' # Optional default '' (stdout) |
||||
# level: 'info' # Optional default 'info' |
||||
|
||||
# MQTT All values default to paho.mqtt.client defaults |
||||
mqtt: |
||||
host: '192.168.88.98' # Optional default 'localhost' |
||||
# port: 1883 # Optional default '1883' |
||||
# keepalive: 60 # Optional |
||||
# auth: # Optional If included, username_pw_set() is called with user/password |
||||
# username: 'user' # Required (when auth is present) |
||||
# password: 'pass' # Optional |
||||
# tls: # Optional If included, tls_set() is called with the following: |
||||
# ca_certs: # Optional |
||||
# certfile: # Optional |
||||
# keyfile: # Optional |
||||
# cert_reqs: # Optional |
||||
# tls_version: # Optional |
||||
# ciphers: # Optional |
||||
|
||||
# Prometheus |
||||
#prometheus: |
||||
# exporter_port: # Optional default 9344 |
||||
@ -0,0 +1,21 @@ |
||||
# Example metric definition |
||||
metrics: |
||||
- name: 'battery_soc' # Required(unique, if multiple, only last entry is kept) |
||||
help: 'state of charge in percentage' # Required |
||||
type: 'gauge' # Required ('gauge', 'counter', 'summary' or 'histogram') |
||||
topic: 'N/+/battery/512/Soc' |
||||
|
||||
- name: 'grid_energy_forward' |
||||
help: 'imported energy in kWh' |
||||
type: 'gauge' |
||||
topic: 'N/+/grid/30/Ac/Energy/Forward' |
||||
|
||||
- name: 'grid_energy_reverse' |
||||
help: 'exported energy in kWh' |
||||
type: 'gauge' |
||||
topic: 'N/+/grid/30/Ac/Energy/Reverse' |
||||
|
||||
- name: 'grid_power' |
||||
help: 'grid power in Watts' |
||||
type: 'gauge' |
||||
topic: 'N/+/grid/30/Ac/Power' |
||||
@ -0,0 +1,26 @@ |
||||
# Config file for MQTT prometheus exporter |
||||
|
||||
# Logging |
||||
#logging: |
||||
# logfile: '' # Optional default '' (stdout) |
||||
# level: 'info' # Optional default 'info' |
||||
|
||||
# MQTT All values default to paho.mqtt.client defaults |
||||
#mqtt: |
||||
# host: 'mqtt.example.com' # Optional default 'localhost' |
||||
# port: 1883 # Optional default '1883' |
||||
# keepalive: 60 # Optional |
||||
# auth: # Optional If included, username_pw_set() is called with user/password |
||||
# username: 'user' # Required (when auth is present) |
||||
# password: 'pass' # Optional |
||||
# tls: # Optional If included, tls_set() is called with the following: |
||||
# ca_certs: # Optional |
||||
# certfile: # Optional |
||||
# keyfile: # Optional |
||||
# cert_reqs: # Optional |
||||
# tls_version: # Optional |
||||
# ciphers: # Optional |
||||
|
||||
# Prometheus |
||||
#prometheus: |
||||
# exporter_port: # Optional default 9344 |
||||
@ -0,0 +1,23 @@ |
||||
# histogram metric. with Buckets <= 0.5, 5, 10, +inf |
||||
- name: 'network_ping_ms' |
||||
help: 'ping response in ms' |
||||
type: 'histogram' |
||||
topic: 'network/+/+/ping' |
||||
parameters: |
||||
buckets: |
||||
- 0.5 |
||||
- 5 |
||||
- 10 |
||||
label_configs: |
||||
- source_labels: ['__msg_topic__'] |
||||
target_label: '__topic__' |
||||
- source_labels: ["__msg_topic__"] |
||||
regex: "network/([^/]+).*" |
||||
target_label: "network" |
||||
replacement: '\1' |
||||
action: "replace" |
||||
- source_labels: ["__msg_topic__"] |
||||
regex: "network/[^/]+/([^/]+).*" |
||||
target_label: "server" |
||||
replacement: '\1' |
||||
action: "replace" |
||||
@ -0,0 +1,27 @@ |
||||
# Example metric definition |
||||
metrics: |
||||
- name: 'mqtt_example' # Required(unique, if multiple, only last entry is kept) |
||||
help: 'MQTT example gauge' # Required |
||||
type: 'gauge' # Required ('gauge', 'counter', 'summary' or 'histogram') |
||||
#parameters: # Optional parameters for certain metrics |
||||
# buckets: # Optional (Passed as 'buckets' argument to Histogram) |
||||
# - .1 |
||||
# - 1.0 |
||||
# - 10.0 |
||||
# states: # Optional (Passes as 'states' arguments to Enum) |
||||
# - on |
||||
# - off |
||||
topic: 'example/topic/+' # Required |
||||
|
||||
# Inspired by 'https://prometheus.io/docs/operating/configuration/#<relabel_config>' |
||||
# "__msg_topic__" and "__value__" are populated with msg topic and value. And "__topic__" is 'topic' from config. |
||||
# Supported actions are: 'replace', 'keep' and 'drop' |
||||
# All labels starting with "__" will be removed, and "__topic__" and "__value__" is copied into "topic" anv "value" |
||||
# after all label configs have been applied. |
||||
label_configs: # Optional |
||||
- source_labels: ['__msg_topic__'] # Required (when label_configs is present) |
||||
separator: '/' # Optional default ';' |
||||
regex: '(.*)' # Optional default '(.*)' |
||||
target_label: '__topic__' # Required (when label_configs is present and 'action' = 'replace') |
||||
replacement: '\1' # Optional default '\1' |
||||
action: 'replace' # Optional default 'replace' |
||||
@ -0,0 +1,61 @@ |
||||
# Config file for Mosquitto broker system metrics |
||||
|
||||
# Metric definitions |
||||
metrics: |
||||
- name: 'mqtt_broker' |
||||
help: 'System events from broker' |
||||
type: 'gauge' |
||||
topic: '$SYS/broker/#' |
||||
label_configs: |
||||
- source_labels: ['__msg_topic__'] |
||||
target_label: '__topic__' |
||||
- source_labels: ['__value__'] |
||||
regex: '^(\d+([,.]\d*)?)$|^([,.]\d+)$' |
||||
action: 'keep' |
||||
|
||||
- name: 'mqtt_broker_version' |
||||
help: 'Mosquitto version (static)' |
||||
type: 'gauge' |
||||
topic: '$SYS/broker/version' |
||||
label_configs: |
||||
- source_labels: ['__msg_topic__'] |
||||
target_label: '__topic__' |
||||
- source_labels: ['__value__'] |
||||
regex: '^\D+((?:\d+[\.]?)+)$' |
||||
target_label: 'version' |
||||
replacement: '\1' |
||||
action: 'replace' |
||||
- source_labels: ['__value__'] |
||||
replacement: '1' |
||||
target_label: '__value__' |
||||
action: 'replace' |
||||
|
||||
- name: 'mqtt_broker_changeset' |
||||
help: 'Mosquitto build changeset (static)' |
||||
type: 'gauge' |
||||
topic: '$SYS/broker/changeset' |
||||
label_configs: |
||||
- source_labels: ['__msg_topic__'] |
||||
target_label: '__topic__' |
||||
- source_labels: ['__value__'] |
||||
target_label: 'changeset' |
||||
action: 'replace' |
||||
- source_labels: ['__value__'] |
||||
replacement: '1' |
||||
target_label: '__value__' |
||||
action: 'replace' |
||||
|
||||
- name: 'mqtt_broker_timestamp' |
||||
help: 'Mosquitto build timestamp (static)' |
||||
type: 'gauge' |
||||
topic: '$SYS/broker/timestamp' |
||||
label_configs: |
||||
- source_labels: ['__msg_topic__'] |
||||
target_label: '__topic__' |
||||
- source_labels: ['__value__'] |
||||
target_label: 'timestamp' |
||||
action: 'replace' |
||||
- source_labels: ['__value__'] |
||||
replacement: '1' |
||||
target_label: '__value__' |
||||
action: 'replace' |
||||
@ -0,0 +1,23 @@ |
||||
# metric for a switch with the state on and off. |
||||
# states are case sensitive and must match exactly |
||||
# use label_config to rewrite other values, see below. |
||||
metrics: |
||||
- name: "fhem_light_state" |
||||
help: "Light state on/off" |
||||
type: "enum" |
||||
topic: "fhem/+/+/light" |
||||
parameters: |
||||
states: |
||||
- 'on' |
||||
- 'off' |
||||
label_configs: |
||||
- source_labels: ['__value__'] # replace uppercase ON and 0 with on |
||||
regex: "(ON|0)" |
||||
target_label: '__value__' |
||||
replacement: 'on' |
||||
action: "replace" |
||||
- source_labels: ['__value__'] # replace uppercase OFF und 1 with off |
||||
regex: "(OFF|1)" |
||||
target_label: '__value__' |
||||
replacement: 'off' |
||||
action: "replace" |
||||
@ -0,0 +1,5 @@ |
||||
paho-mqtt |
||||
prometheus-client |
||||
PyYAML |
||||
yamlreader |
||||
pytest |
||||
@ -0,0 +1,5 @@ |
||||
paho-mqtt==1.5.1 |
||||
prometheus-client==0.11.0 |
||||
PyYAML==5.4.1 |
||||
six==1.16.0 |
||||
yamlreader==3.0.4 |
||||
@ -0,0 +1,4 @@ |
||||
paho-mqtt |
||||
prometheus-client |
||||
PyYAML |
||||
yamlreader |
||||
@ -0,0 +1,74 @@ |
||||
# Tests with pytest |
||||
|
||||
make sure pytest is installed `pip install -r requirements-dev.txt` |
||||
run `pytest -s -o log_cli=true -o log_cli_level="DEBUG"` from the repository root |
||||
|
||||
## test_mqtt_explorer.py:test_update_metrics |
||||
|
||||
This test loads test mqtt data from a file and feeds it into mqtt_exporter and check if expected results are recorded in the prometheus client. |
||||
|
||||
### directory structure |
||||
Test data is loaded from following directory structure: |
||||
|
||||
``` |
||||
./tests/ |
||||
./test_data/ |
||||
./test1/ |
||||
conf.yaml |
||||
mqtt_msg.csv |
||||
./test2/ |
||||
conf.yaml |
||||
mqtt_msg.csv |
||||
./test_xyz/ |
||||
conf.yaml |
||||
mqtt_msg.csv |
||||
./tmp_data |
||||
[metric_test...##,txt] |
||||
``` |
||||
|
||||
In `test-data` each subfolder (e.g. `test1`, `test_bla`) contains a separate set of test data. There is no naming convention for folders. The could be descriptive like `test_for_issue1234`. Avoid any special characters and white spaces. |
||||
|
||||
Files: |
||||
- `conf.yaml`: a config like a config for mqtt_exporter itself, but only the `metrics` part and a new optional attribute `timescale` is read from it. |
||||
- `mqtt_msg.csv`: fake mqtt data, format description see below. |
||||
|
||||
`tmp-data` contains prometheus scrape output from after each processed mqtt msg data. This folder will be cleaned before each test run. |
||||
|
||||
### mqtt_msg.csv file format |
||||
|
||||
`mqtt_msg.csv` is a CSV file with `;` as delimiter and `'` as quotation character. |
||||
Following Column looks are expected: |
||||
|
||||
``` |
||||
in_topic;in_payload;out_name;out_labels;out_value;delay;assert |
||||
``` |
||||
- `in_topic`: topic as from mqtt server |
||||
- `in_payload`: payload from mqtt server as string (will be converted byte array) |
||||
- `out_name`: metric name without any suffix like `total`, `sum`, `bucket`, ... |
||||
- `out_labels`: labels notes as a JSON string including the topic. |
||||
- `out_value`: expected value for simple metrics like gauge it is a number. For other metrics is is a JSON string with expected values per suffix e.g. `{"_count": 10, "_sum": 85.55, "_bucket": 10}` |
||||
- `delay`: seconds delay until the next mqtt msg is processed. The `timescale` config attribute speed up/slow down the delay. A time scale of 0 means no default, a timescale of 1 means realtime. Default timescale = 0 |
||||
- `assert`: `True/False`. Specify if the test should pass or not. In most cases this should be `True` |
||||
|
||||
Metric type `Histogram` special handling here as it will log a `$(metric_name)_bucket` metric for each bucket with a reserved label `le` in the meaning of _less or equal_. Specify `le` for one bucket and set the expected count to the `bucket` attribute in the `out_value` JSON. See examples in `test1`. |
||||
|
||||
For sample data see existing tests above. |
||||
|
||||
### Gather test data from live environment |
||||
|
||||
If logging level is set to `debug` the log will contain some lines that should be already correct formatted to be placed in a `mqtt_msq.csv`. |
||||
|
||||
they look like this: |
||||
``` |
||||
2021-08-08 22:24:36,996 DEBUG: TEST_DATA: fhem/Terrasse/TermPearl02/humidity; 21.0; fhem_humidity_percent; {"location": "paz", "topic": "fhem/paz/TermPearl01/humidity"}; 17.0; 0; True |
||||
2021-08-08 22:24:30,601 DEBUG: TEST_DATA: fhem/Terrasse/TerrasseWeiss/humidity; 20.0; fhem_humidity_percent; {"location": "paz", "topic": "fhem/paz/TermPearl01/humidity"}; 17.0; 0; True |
||||
2021-08-08 22:24:27,097 DEBUG: TEST_DATA: fhem/Garten/TermFetanten01/temperature; 16.7; fhem_temperature_celsius; {"location": "Terrasse", "topic": "fhem/Terrasse/TerrasseWeiss/temperature"}; 16.0; 0; True |
||||
2021-08-08 22:23:58,831 DEBUG: TEST_DATA: fhem/paz/TermPearl01/humidity; 17.0; fhem_humidity_percent; {"location": "paz", "topic": "fhem/paz/TermPearl01/humidity"}; 17.0; 0; True |
||||
``` |
||||
Tips: |
||||
- remove `.* DEBUG: TEST_DATA: `. |
||||
- Make sure the `mqtt_msg.csv` contains as first line the headers given above. |
||||
- The captured data won't fit if the `payload/__value__` has been replaced by a label_config. Please set `in_payload` to the correct value manually. An example for this exception is the `- name: 'mqtt_broker_version'` metric from the example configurations. |
||||
- put the data in a new subfolder in the `test_data` dir. Copy also the config file from the live environment to this folder (you should remove the `mqtt` part from it. Make also sure the recorded data don't contain sensitive data). |
||||
- Create a PR and share the test data, as this will allow all developers to verify code changes. |
||||
|
||||
@ -0,0 +1,76 @@ |
||||
|
||||
# Logging |
||||
logging: |
||||
# logfile: 'conf/mqttexperter.log' # Optional default '' (stdout) |
||||
level: 'debug' # Optional default 'info' |
||||
|
||||
timescale: 0 |
||||
|
||||
# Metric definitions |
||||
metrics: |
||||
- name: 'ftp_transferred_bytes' |
||||
help: 'data transferred in bytes pe file' |
||||
type: 'summary' |
||||
topic: 'ftp/+/transferred' |
||||
label_configs: |
||||
- source_labels: ['__msg_topic__'] |
||||
target_label: '__topic__' |
||||
- source_labels: ["__msg_topic__"] |
||||
regex: "ftp/([^/]+).*" |
||||
target_label: "file" |
||||
replacement: '\1' |
||||
action: "replace" |
||||
- name: 'network_ping_ms' |
||||
help: 'ping response in ms' |
||||
type: 'histogram' |
||||
topic: 'network/+/+/ping' |
||||
buckets: '0.5,5,10' |
||||
label_configs: |
||||
- source_labels: ['__msg_topic__'] |
||||
target_label: '__topic__' |
||||
- source_labels: ["__msg_topic__"] |
||||
regex: "network/([^/]+).*" |
||||
target_label: "network" |
||||
replacement: '\1' |
||||
action: "replace" |
||||
- source_labels: ["__msg_topic__"] |
||||
regex: "network/[^/]+/([^/]+).*" |
||||
target_label: "server" |
||||
replacement: '\1' |
||||
action: "replace" |
||||
- name: "fhem_temperature_celsius" |
||||
help: "443 Mhz Sensors, Temperature in C" |
||||
type: "gauge" |
||||
topic: "fhem/+/+/temperature" |
||||
label_configs: |
||||
- source_labels: ['__msg_topic__'] |
||||
target_label: '__topic__' |
||||
- source_labels: ["__msg_topic__"] |
||||
regex: "fhem/([^/]+).*" |
||||
target_label: "location" |
||||
replacement: '\1' |
||||
action: "replace" |
||||
- name: "fhem_humidity_percent" |
||||
help: "443 Mhz Sensors, Humidity in %" |
||||
type: "gauge" |
||||
topic: "fhem/+/+/humidity" |
||||
label_configs: |
||||
- source_labels: ['__msg_topic__'] |
||||
target_label: '__topic__' |
||||
- source_labels: ["__msg_topic__"] |
||||
regex: "fhem/([^/]+).*" |
||||
target_label: "location" |
||||
replacement: '\1' |
||||
action: "replace" |
||||
- name: "fhem_rain_mm" |
||||
help: "443 Mhz Sensors, rain in mm/m2" |
||||
type: "counter" |
||||
topic: "fhem/+/+/rain_total" |
||||
label_configs: |
||||
- source_labels: ['__msg_topic__'] |
||||
target_label: '__topic__' |
||||
- source_labels: ["__msg_topic__"] |
||||
regex: "fhem/([^/]+).*" |
||||
target_label: "location" |
||||
replacement: '\1' |
||||
action: "replace" |
||||
|
unable to load file from base commit
|
@ -0,0 +1,99 @@ |
||||
# Config file for MQTT prometheus exporter |
||||
|
||||
|
||||
# Metric definitions |
||||
metrics: |
||||
# - name: 'mqtt_broker' |
||||
# help: 'System events from broker' |
||||
# type: 'gauge' |
||||
# topic: '$SYS/broker/#' |
||||
# label_configs: |
||||
# - source_labels: ['__msg_topic__'] |
||||
# target_label: '__topic__' |
||||
# - source_labels: ['__value__'] |
||||
# regex: '^(\d+([,.]\d*)?)$|^([,.]\d+)$' |
||||
# action: 'keep' |
||||
|
||||
- name: 'mqtt_broker_version' |
||||
help: 'Mosquitto version (static)' |
||||
type: 'gauge' |
||||
topic: '$SYS/broker/version' |
||||
label_configs: |
||||
- source_labels: ['__msg_topic__'] |
||||
target_label: '__topic__' |
||||
- source_labels: ['__value__'] |
||||
regex: '^\D+((?:\d+[\.]?)+)$' |
||||
target_label: 'version' |
||||
replacement: '\1' |
||||
action: 'replace' |
||||
- source_labels: ['__value__'] |
||||
replacement: '1' |
||||
target_label: '__value__' |
||||
action: 'replace' |
||||
|
||||
- name: 'mqtt_broker_changeset' |
||||
help: 'Mosquitto build changeset (static)' |
||||
type: 'gauge' |
||||
topic: '$SYS/broker/changeset' |
||||
label_configs: |
||||
- source_labels: ['__msg_topic__'] |
||||
target_label: '__topic__' |
||||
- source_labels: ['__value__'] |
||||
target_label: 'changeset' |
||||
action: 'replace' |
||||
- source_labels: ['__value__'] |
||||
replacement: '1' |
||||
target_label: '__value__' |
||||
action: 'replace' |
||||
|
||||
- name: 'mqtt_broker_timestamp' |
||||
help: 'Mosquitto build timestamp (static)' |
||||
type: 'gauge' |
||||
topic: '$SYS/broker/timestamp' |
||||
label_configs: |
||||
- source_labels: ['__msg_topic__'] |
||||
target_label: '__topic__' |
||||
- source_labels: ['__value__'] |
||||
target_label: 'timestamp' |
||||
action: 'replace' |
||||
- source_labels: ['__value__'] |
||||
replacement: '1' |
||||
target_label: '__value__' |
||||
action: 'replace' |
||||
|
||||
- name: "fhem_temperature_celsius" |
||||
help: "443 Mhz Sensors, Temperature in C" |
||||
type: "gauge" |
||||
topic: "fhem/+/+/temperature" |
||||
label_configs: |
||||
- source_labels: ['__msg_topic__'] |
||||
target_label: '__topic__' |
||||
- source_labels: ["__msg_topic__"] |
||||
regex: "fhem/([^/]+).*" |
||||
target_label: "location" |
||||
replacement: '\1' |
||||
action: "replace" |
||||
- name: "fhem_humidity_percent" |
||||
help: "443 Mhz Sensors, Humidity in %" |
||||
type: "gauge" |
||||
topic: "fhem/+/+/humidity" |
||||
label_configs: |
||||
- source_labels: ['__msg_topic__'] |
||||
target_label: '__topic__' |
||||
- source_labels: ["__msg_topic__"] |
||||
regex: "fhem/([^/]+).*" |
||||
target_label: "location" |
||||
replacement: '\1' |
||||
action: "replace" |
||||
- name: "fhem_rain_mm" |
||||
help: "443 Mhz Sensors, rain in mm/m2" |
||||
type: "counter" |
||||
topic: "fhem/+/+/rain_total" |
||||
label_configs: |
||||
- source_labels: ['__msg_topic__'] |
||||
target_label: '__topic__' |
||||
- source_labels: ["__msg_topic__"] |
||||
regex: "fhem/([^/]+).*" |
||||
target_label: "location" |
||||
replacement: '\1' |
||||
action: "replace" |
||||
|
unable to load file from base commit
|
@ -0,0 +1,22 @@ |
||||
|
||||
# Logging |
||||
logging: |
||||
# logfile: 'conf/mqttexperter.log' # Optional default '' (stdout) |
||||
level: 'debug' # Optional default 'info' |
||||
|
||||
timescale: 0 |
||||
|
||||
# Metric definitions |
||||
metrics: |
||||
- name: "fhem_rain_mm" |
||||
help: "443 Mhz Sensors, rain in mm/m2" |
||||
type: "counter_absolute" |
||||
topic: "fhem/+/+/rain_total" |
||||
label_configs: |
||||
- source_labels: ['__msg_topic__'] |
||||
target_label: '__topic__' |
||||
- source_labels: ["__msg_topic__"] |
||||
regex: "fhem/([^/]+).*" |
||||
target_label: "location" |
||||
replacement: '\1' |
||||
action: "replace" |
||||
|
unable to load file from base commit
|
@ -0,0 +1,58 @@ |
||||
|
||||
# Logging |
||||
logging: |
||||
# logfile: 'conf/mqttexperter.log' # Optional default '' (stdout) |
||||
level: 'debug' # Optional default 'info' |
||||
|
||||
timescale: 0 |
||||
|
||||
# Metric definitions |
||||
metrics: |
||||
- name: "fhem_light_state" |
||||
help: "Light state on/off" |
||||
type: "enum" |
||||
topic: "fhem/+/+/light" |
||||
parameters: |
||||
states: |
||||
- 'on' |
||||
- 'off' |
||||
label_configs: |
||||
- source_labels: ['__value__'] |
||||
regex: "(ON|0)" |
||||
target_label: '__value__' |
||||
replacement: 'on' |
||||
action: "replace" |
||||
- source_labels: ['__value__'] |
||||
regex: "(OFF|1)" |
||||
target_label: '__value__' |
||||
replacement: 'off' |
||||
action: "replace" |
||||
- source_labels: ['__msg_topic__'] |
||||
target_label: '__topic__' |
||||
- source_labels: ["__msg_topic__"] |
||||
regex: "fhem/([^/]+).*" |
||||
target_label: "location" |
||||
replacement: '\1' |
||||
action: "replace" |
||||
- name: 'network_ping_ms' |
||||
help: 'ping response in ms' |
||||
type: 'histogram' |
||||
topic: 'network/+/+/ping' |
||||
parameters: |
||||
buckets: |
||||
- 0.5 |
||||
- 5 |
||||
- 10 |
||||
label_configs: |
||||
- source_labels: ['__msg_topic__'] |
||||
target_label: '__topic__' |
||||
- source_labels: ["__msg_topic__"] |
||||
regex: "network/([^/]+).*" |
||||
target_label: "network" |
||||
replacement: '\1' |
||||
action: "replace" |
||||
- source_labels: ["__msg_topic__"] |
||||
regex: "network/[^/]+/([^/]+).*" |
||||
target_label: "server" |
||||
replacement: '\1' |
||||
action: "replace" |
||||
|
unable to load file from base commit
|
@ -0,0 +1,178 @@ |
||||
""" |
||||
py test testing for mqtt_exporter |
||||
|
||||
""" |
||||
|
||||
import csv |
||||
import distutils.util |
||||
import json |
||||
from json.decoder import JSONDecodeError |
||||
import os |
||||
import time |
||||
import logging |
||||
|
||||
import prometheus_client as prometheus |
||||
import prometheus_client.registry |
||||
import mqtt_exporter |
||||
import pytest |
||||
|
||||
logging.basicConfig(level=logging.DEBUG) |
||||
|
||||
TMP_DIR=os.path.join( |
||||
os.path.dirname(__file__), |
||||
'tmp_data' |
||||
) |
||||
DATA_DIR=os.path.join( |
||||
os.path.dirname(__file__), |
||||
'test_data' |
||||
) |
||||
|
||||
def setup_module(module): #pylint: disable=unused-argument |
||||
""" setup any state specific to the execution of the given module.""" |
||||
delete_temp_test_files() |
||||
|
||||
|
||||
def delete_temp_test_files(): |
||||
# delete TEMP files |
||||
for file in os.listdir(TMP_DIR): |
||||
if file == '.gitkeep': |
||||
continue |
||||
os.remove(os.path.join(TMP_DIR, file)) |
||||
|
||||
|
||||
class MqttCVS: |
||||
in_topic = "in_topic" |
||||
in_payload = "in_payload" |
||||
out_name = "out_name" |
||||
out_labels = "out_labels" |
||||
out_value = "out_value" |
||||
delay = "delay" |
||||
expected_assert = "assert" |
||||
|
||||
def _get_mqtt_data(file_name): |
||||
""" |
||||
Reads mqtt fake data and expected results from file |
||||
""" |
||||
mqtt_data = [] |
||||
with open(file_name, newline='') as mqtt_data_csv: |
||||
csv_reader = csv.DictReader(mqtt_data_csv, quotechar="'", delimiter=';') |
||||
for row in csv_reader: |
||||
row[MqttCVS.in_topic] = row[MqttCVS.in_topic].strip() |
||||
row[MqttCVS.out_name] = row[MqttCVS.out_name].strip() |
||||
# covert payloud to bytes, as in a MQTT Message |
||||
row[MqttCVS.in_payload] = row[MqttCVS.in_payload].encode('UTF-8') |
||||
# parse labels, to a python object. |
||||
try: |
||||
row[MqttCVS.out_labels] = json.loads(row.get(MqttCVS.out_labels, '{}')) |
||||
except json.decoder.JSONDecodeError as jde: |
||||
logging.error(f"json.decoder.JSONDecodeError while decoding {row.get(MqttCVS.out_labels, '{}')}") |
||||
raise jde |
||||
# Value could be a JSON, a float or anthing else. |
||||
try: |
||||
row[MqttCVS.out_value] = float(row.get(MqttCVS.out_value)) |
||||
except ValueError: |
||||
try: |
||||
row[MqttCVS.out_value] = json.loads(row.get(MqttCVS.out_value)) |
||||
except (JSONDecodeError, TypeError): |
||||
pass # leave as it is |
||||
# set delay to 0 if not a number |
||||
try: |
||||
row[MqttCVS.delay] = float(row.get(MqttCVS.delay, 0)) |
||||
except ValueError: |
||||
row[MqttCVS.delay] = 0 |
||||
# convert string to bool for expected assertion. |
||||
row[MqttCVS.expected_assert] = bool( |
||||
distutils.util.strtobool(row.get(MqttCVS.expected_assert, "True").strip())) |
||||
mqtt_data.append(row) |
||||
return mqtt_data |
||||
|
||||
|
||||
def _get_test_data(): |
||||
""" |
||||
Reads test data from DATA_DIR sub directories. |
||||
Each subdirectory is expected to contain a `conf.yaml` file with a metrics config (like in the config file) |
||||
and a CSV file `mqtt_msg.csv` with fake mqtt data ";" delimited: |
||||
`in_topic;in_payload;out_name;out_labels;out_value;delay;assert` |
||||
where |
||||
lables_out: json string with all expected lables |
||||
delay: delay until the next line is processed |
||||
assert: expected assert result, True if out_value matches prometheus metric |
||||
""" |
||||
test_data_sets = [] |
||||
test_data_dirs = [f.path for f in os.scandir(DATA_DIR) if f.is_dir()] |
||||
test_names = [ os.path.basename(os.path.normpath(name)) for name in test_data_dirs] |
||||
for test_data_dir in test_data_dirs: |
||||
conf_file = os.path.join(test_data_dir, 'conf.yaml') |
||||
mqtt_data_file = os.path.join(test_data_dir, 'mqtt_msg.csv') |
||||
if not os.path.isfile(conf_file) or not os.path.isfile(mqtt_data_file): |
||||
logging.error(f"Test data dir {test_data_dir} doesn't contain required files, skipping") |
||||
continue |
||||
config_yaml = mqtt_exporter._read_config(conf_file) |
||||
config_yaml = mqtt_exporter._parse_config_and_add_defaults(config_yaml) |
||||
test_data_sets.append(( |
||||
config_yaml['metrics'], |
||||
_get_mqtt_data(mqtt_data_file), |
||||
config_yaml.get('timescale', 0), |
||||
)) |
||||
return test_names, test_data_sets |
||||
|
||||
def _get_suffixes_by_metric_name(metrics, metric_name): |
||||
metric_type = None |
||||
for _, outer_metric in metrics.items(): |
||||
for metric in outer_metric: |
||||
if metric['name'] == metric_name: |
||||
metric_type = metric['type'] |
||||
break |
||||
|
||||
for suffix in mqtt_exporter.SUFFIXES_PER_TYPE[metric_type]: |
||||
if len(suffix) == 0: |
||||
yield suffix |
||||
else: |
||||
yield f"_{suffix}" |
||||
|
||||
|
||||
class FakeMSG(): |
||||
""""Simulate MQTT Msg""" |
||||
def __init__(self, topic, payload) -> None: |
||||
self.topic = topic |
||||
self.payload = payload |
||||
|
||||
|
||||
param_test_data_dirs, param_test_data_sets = _get_test_data() |
||||
|
||||
@pytest.mark.parametrize("metrics,mqtt_data_set,timescale", param_test_data_sets, ids=param_test_data_dirs) |
||||
def test_update_metrics(caplog, request, metrics, mqtt_data_set, timescale): |
||||
""" |
||||
reads a label_config and some mqtt data and asserts if they are in the metrics |
||||
""" |
||||
logging.info(f"Start test_update_metrics with ID {request.node.callspec.id}") |
||||
|
||||
# reset prometheus registry between tests |
||||
collectors = list(prometheus.REGISTRY._collector_to_names.keys()) |
||||
for collector in collectors: |
||||
prometheus.REGISTRY.unregister(collector) |
||||
|
||||
i = 1 |
||||
for mqtt_data in mqtt_data_set: |
||||
msg = FakeMSG(mqtt_data[MqttCVS.in_topic], mqtt_data[MqttCVS.in_payload]) |
||||
mqtt_exporter._on_message(None, metrics, msg) |
||||
prometheus.REGISTRY.collect() |
||||
prometheus.write_to_textfile(os.path.join(TMP_DIR, f"metric_{request.node.callspec.id}_{i:02}.txt"), prometheus.REGISTRY) |
||||
# depending on metric type one or more metrics with different suffixes are added. |
||||
for suffix in _get_suffixes_by_metric_name(metrics, mqtt_data[MqttCVS.out_name]): |
||||
# historgram with buckets need special handling, remove bucket labe label 'le' |
||||
labels = mqtt_data[MqttCVS.out_labels].copy() |
||||
if not suffix == "_bucket" and labels.get('le'): |
||||
labels.pop('le') |
||||
|
||||
expected_result = mqtt_data[MqttCVS.out_value] |
||||
expected_result = expected_result if not isinstance(expected_result, dict) else expected_result[suffix] |
||||
logging.info(f"Assert {mqtt_data[MqttCVS.out_name]}{suffix} from testdata record {i}") |
||||
assert ( prometheus.REGISTRY.get_sample_value( |
||||
f"{mqtt_data[MqttCVS.out_name]}{suffix}", |
||||
labels |
||||
) == expected_result ) == mqtt_data[MqttCVS.expected_assert] |
||||
time.sleep(mqtt_data[MqttCVS.delay] * timescale) |
||||
i += 1 |
||||
for record in caplog.records: |
||||
assert record.levelno < logging.ERROR |
||||
@ -0,0 +1,72 @@ |
||||
""" |
||||
Pytest for prometheus client enhancements |
||||
""" |
||||
import os |
||||
import logging |
||||
import time |
||||
import pytest |
||||
from utils.prometheus_additions import CounterAbsolute |
||||
import prometheus_client as prometheus |
||||
|
||||
logging.basicConfig(level=logging.DEBUG) |
||||
|
||||
TMP_DIR=os.path.join( |
||||
os.path.dirname(__file__), |
||||
'tmp_data' |
||||
) |
||||
DATA_DIR=os.path.join( |
||||
os.path.dirname(__file__), |
||||
'test_data' |
||||
) |
||||
|
||||
@pytest.fixture(scope="class") |
||||
def get_registry(): |
||||
yield prometheus.REGISTRY |
||||
# reset prometheus registry between tests |
||||
collectors = list(prometheus.REGISTRY._collector_to_names.keys()) |
||||
for collector in collectors: |
||||
prometheus.REGISTRY.unregister(collector) |
||||
|
||||
old_creation_time = 0.0 |
||||
|
||||
class TestCounterWithReset: |
||||
a_counter_absolute = CounterAbsolute('Absolute_Counter', 'Test metric' ) |
||||
old_creation_time = 0.0 |
||||
param_test_data_sets = [ |
||||
(10, False), |
||||
(10, True), |
||||
(11, True), |
||||
(110, True), |
||||
(110, True), |
||||
(210, True), |
||||
(310.7, True), |
||||
(110, False), |
||||
(210, True), |
||||
(310.7, True), |
||||
] |
||||
|
||||
@pytest.mark.parametrize("value, same_creation_time", param_test_data_sets) |
||||
def test_counter_absolute(self, request, get_registry, value, same_creation_time): |
||||
global old_creation_time |
||||
self.a_counter_absolute.set(value) |
||||
creation_time = self.a_counter_absolute._created |
||||
logging.info(f"Creation time: {creation_time:e}") |
||||
registry = get_registry |
||||
registry.collect() |
||||
prometheus.write_to_textfile(os.path.join(TMP_DIR, f"absolute_counter_{request.node.callspec.id}_{value:05}.txt"), prometheus.REGISTRY) |
||||
|
||||
assert self.a_counter_absolute._value.get() == value |
||||
assert (creation_time == old_creation_time ) == same_creation_time |
||||
old_creation_time = creation_time |
||||
time.sleep(0.005) |
||||
|
||||
|
||||
class TestCounterRestForbidden: |
||||
a_counter_absolute = CounterAbsolute('Strict_Absolute_Counter', "This Counters doesn't allow reset") |
||||
|
||||
def test_counter_reset(self): |
||||
val_first = 0.3324234 |
||||
val_second = 0.3324233 |
||||
self.a_counter_absolute.set(val_first, fail_on_decrease=True) |
||||
with pytest.raises(ValueError, match=rf"Counter must increase {val_second} lower {val_first}"): |
||||
self.a_counter_absolute.set(val_second, fail_on_decrease=True) |
||||
Binary file not shown.
Binary file not shown.
@ -0,0 +1,40 @@ |
||||
""" |
||||
Additions and enhancements to the prometheus_client package |
||||
""" |
||||
|
||||
import prometheus_client as prometheus |
||||
|
||||
class CounterAbsolute(prometheus.Counter): |
||||
""" |
||||
CounterAbsolute allows to set the Counter by an absolute value like Gauge, but data is |
||||
handled properly if counter resets or over flows. |
||||
CounterAbsolute is typically used if values need to by proxied from another source e.g. |
||||
a network counter, SMTP or MQTT data which return increasing but absolute numbers |
||||
instead of a diff. |
||||
|
||||
As Counter must not decrease, setting CounterAbsolute to a lower value is handled as follows: |
||||
A counter overflow or a reset is assumed and the create timestamp gets reset and internally |
||||
a new Value object created. |
||||
|
||||
An example for a CounterAbsolute: |
||||
from prometheus_client import Counter |
||||
c = CounterAbsolute('my_failures_total', 'Description of counter') |
||||
c.set(1123.63213) # Set to an absolute value. If lower than last value, Counter gets reset. |
||||
|
||||
""" |
||||
_type = 'counter' |
||||
|
||||
|
||||
def set(self, value, fail_on_decrease=False): |
||||
"""Increment counter to the given amount.""" |
||||
self._raise_if_not_observable() |
||||
if value < 0: |
||||
raise ValueError('Counters can be a positive number only.') |
||||
if value >= self._value.get(): |
||||
self._value.set(float(value)) |
||||
else: |
||||
if fail_on_decrease: |
||||
raise ValueError(f"Counter must increase {value} lower {self._value.get()}") |
||||
else: |
||||
self._metric_init() |
||||
self._value.set(float(value)) |
||||
@ -0,0 +1,4 @@ |
||||
""" |
||||
Version of the whole project. |
||||
""" |
||||
__version__ = "1.0.0" |
||||
@ -0,0 +1,649 @@ |
||||
#!/usr/bin/env python |
||||
|
||||
''' |
||||
A purposely build Prometheus exporter for Victron Energy Cerbo GX MQTT devices |
||||
|
||||
(C) 2022 M. Konstapel https://meezenest.nl/mees |
||||
|
||||
Based on the General purpose Prometheus exporter for MQTT from Frederic Hemberger |
||||
(https://github.com/fhemberger/mqtt_exporter) |
||||
|
||||
MIT License |
||||
|
||||
Copyright (c) 2018 Bendik Wang Andreassen |
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy |
||||
of this software and associated documentation files (the "Software"), to deal |
||||
in the Software without restriction, including without limitation the rights |
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell |
||||
copies of the Software, and to permit persons to whom the Software is |
||||
furnished to do so, subject to the following conditions: |
||||
|
||||
The above copyright notice and this permission notice shall be included in all |
||||
copies or substantial portions of the Software. |
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE |
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER |
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, |
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE |
||||
SOFTWARE. |
||||
''' |
||||
|
||||
from copy import deepcopy |
||||
import json |
||||
from collections import defaultdict |
||||
import logging |
||||
import argparse |
||||
import os |
||||
import re |
||||
import operator |
||||
import time |
||||
import signal |
||||
import sys |
||||
import paho.mqtt.client as mqtt |
||||
from paho.mqtt.properties import Properties |
||||
from paho.mqtt.packettypes import PacketTypes |
||||
import yaml |
||||
import prometheus_client as prometheus |
||||
from yamlreader import yaml_load |
||||
import utils.prometheus_additions |
||||
import version |
||||
|
||||
VERSION = version.__version__ |
||||
SUFFIXES_PER_TYPE = { |
||||
"gauge": [], |
||||
"counter": ['total'], |
||||
"counter_absolute": ['total'], |
||||
"summary": ['sum', 'count'], |
||||
"histogram": ['sum', 'count', 'bucket'], |
||||
"enum": [], |
||||
} |
||||
|
||||
|
||||
def _read_config(config_path): |
||||
"""Read config file from given location, and parse properties""" |
||||
|
||||
if config_path is not None: |
||||
if os.path.isfile(config_path): |
||||
logging.info(f'Config file found at: {config_path}') |
||||
try: |
||||
with open(config_path, 'r') as f: |
||||
return yaml.safe_load(f.read()) |
||||
except yaml.YAMLError: |
||||
logging.exception('Failed to parse configuration file:') |
||||
|
||||
elif os.path.isdir(config_path): |
||||
logging.info( |
||||
f'Config directory found at: {config_path}') |
||||
try: |
||||
return yaml_load(config_path) |
||||
except yaml.YAMLError: |
||||
logging.exception('Failed to parse configuration directory:') |
||||
|
||||
return {} |
||||
|
||||
|
||||
def _parse_config_and_add_defaults(config_from_file): |
||||
"""Parse content of configfile and add default values where needed""" |
||||
|
||||
config = deepcopy(config_from_file) |
||||
logging.debug(f'_parse_config Config from file: {str(config_from_file)}') |
||||
# Logging values ('logging' is optional in config |
||||
if 'logging' in config_from_file: |
||||
config['logging'] = _add_config_and_defaults( |
||||
config_from_file['logging'], {'logfile': '', 'level': 'info'}) |
||||
else: |
||||
config['logging'] = _add_config_and_defaults( |
||||
None, {'logfile': '', 'level': 'info'}) |
||||
|
||||
# MQTT values |
||||
if 'mqtt' in config_from_file: |
||||
config['mqtt'] = _add_config_and_defaults( |
||||
config_from_file['mqtt'], {'host': 'localhost', 'port': 1883}) |
||||
else: |
||||
config['mqtt'] = _add_config_and_defaults( |
||||
None, {'host': 'localhost', 'port': 1883}) |
||||
|
||||
if 'auth' in config['mqtt']: |
||||
config['mqtt']['auth'] = _add_config_and_defaults( |
||||
config['mqtt']['auth'], {}) |
||||
_validate_required_fields(config['mqtt']['auth'], 'auth', ['username']) |
||||
|
||||
if 'tls' in config['mqtt']: |
||||
config['mqtt']['tls'] = _add_config_and_defaults( |
||||
config['mqtt']['tls'], {}) |
||||
|
||||
# Prometheus values |
||||
if 'prometheus' in config: |
||||
config['prometheus'] = _add_config_and_defaults( |
||||
config_from_file['prometheus'], {'exporter_port': 9344}) |
||||
else: |
||||
config['prometheus'] = _add_config_and_defaults( |
||||
None, {'exporter_port': 9344}) |
||||
|
||||
metrics = {} |
||||
if not 'metrics' in config_from_file: |
||||
logging.critical('No metrics defined in config. Aborting.') |
||||
sys.exit(1) |
||||
for metric in config_from_file['metrics']: |
||||
parse_and_validate_metric_config(metric, metrics) |
||||
|
||||
config['metrics'] = _group_by_topic(list(metrics.values())) |
||||
return config |
||||
|
||||
|
||||
def parse_and_validate_metric_config(metric, metrics): |
||||
m = _add_config_and_defaults(metric, {}) |
||||
_validate_required_fields(m, None, ['name', 'help', 'type', 'topic']) |
||||
if 'label_configs' in m and m['label_configs']: |
||||
label_configs = [] |
||||
for lc in m['label_configs']: |
||||
if lc: |
||||
lc = _add_config_and_defaults(lc, {'separator': ';', 'regex': '^(.*)$', 'replacement': '\\1', |
||||
'action': 'replace'}) |
||||
if lc['action'] == 'replace': |
||||
_validate_required_fields(lc, None, |
||||
['source_labels', 'target_label', 'separator', 'regex', 'replacement', |
||||
'action']) |
||||
else: |
||||
_validate_required_fields(lc, None, |
||||
['source_labels', 'separator', 'regex', 'replacement', |
||||
'action']) |
||||
label_configs.append(lc) |
||||
m['label_configs'] = label_configs |
||||
# legacy config handling move 'buckets' to params directory |
||||
if m.get('buckets'): |
||||
m.setdefault('parameters', {})['buckets'] = (m['buckets']) |
||||
metrics[m['name']] = m |
||||
|
||||
|
||||
def _validate_required_fields(config, parent, required_fields): |
||||
"""Fail if required_fields is not present in config""" |
||||
for field in required_fields: |
||||
if field not in config or config[field] is None: |
||||
if parent is None: |
||||
error = f'\'{field}\' is a required field in configfile' |
||||
else: |
||||
error = f'\'{field}\' is a required parameter for field {parent} in configfile' |
||||
raise TypeError(error) |
||||
|
||||
|
||||
def _add_config_and_defaults(config, defaults): |
||||
"""Return dict with values from config, if present, or values from defaults""" |
||||
if config is not None: |
||||
defaults.update(config) |
||||
return defaults.copy() |
||||
|
||||
|
||||
def _strip_config(config, allowed_keys): |
||||
return {k: v for k, v in config.items() if k in allowed_keys and v} |
||||
|
||||
|
||||
def _group_by_topic(metrics): |
||||
"""Group metrics by topic""" |
||||
t = defaultdict(list) |
||||
for metric in metrics: |
||||
t[metric['topic']].append(metric) |
||||
return t |
||||
|
||||
|
||||
def _topic_matches(topic1, topic2): |
||||
"""Check if wildcard-topics match""" |
||||
if topic1 == topic2: |
||||
return True |
||||
|
||||
# If topic1 != topic2 and no wildcard is present in topic1, no need for regex |
||||
if '+' not in topic1 and '#' not in topic1: |
||||
return False |
||||
|
||||
logging.debug( |
||||
f'_topic_matches: Topic1: {topic1}, Topic2: {topic2}') |
||||
topic1 = re.escape(topic1) |
||||
regex = topic1.replace('/\\#', '.*$').replace('\\+', '[^/]+') |
||||
match = re.match(regex, topic2) |
||||
|
||||
logging.debug(f'_topic_matches: Match: {match is not None}') |
||||
return match is not None |
||||
|
||||
|
||||
# noinspection SpellCheckingInspection |
||||
def _log_setup(logging_config): |
||||
"""Setup application logging""" |
||||
|
||||
logfile = logging_config['logfile'] |
||||
|
||||
log_level = logging_config['level'] |
||||
|
||||
numeric_level = logging.getLevelName(log_level.upper()) |
||||
if not isinstance(numeric_level, int): |
||||
raise TypeError(f'Invalid log level: {log_level}') |
||||
|
||||
if logfile != '': |
||||
logging.info(f"Logging redirected to: {logfile}") |
||||
# Need to replace the current handler on the root logger: |
||||
file_handler = logging.FileHandler(logfile, 'a') |
||||
formatter = logging.Formatter('%(asctime)s %(levelname)s: %(message)s') |
||||
file_handler.setFormatter(formatter) |
||||
|
||||
log = logging.getLogger() # root logger |
||||
for handler in log.handlers: # remove all old handlers |
||||
log.removeHandler(handler) |
||||
log.addHandler(file_handler) |
||||
|
||||
else: |
||||
logging.basicConfig(format='%(asctime)s %(levelname)s: %(message)s') |
||||
|
||||
logging.getLogger().setLevel(numeric_level) |
||||
logging.info(f'log_level set to: {log_level}') |
||||
|
||||
|
||||
# noinspection PyUnusedLocal |
||||
def _on_connect(client, userdata, flags, rc): # pylint: disable=unused-argument,invalid-name |
||||
"""The callback for when the client receives a CONNACK response from the server.""" |
||||
logging.info(f'Connected to broker, result code {str(rc)}') |
||||
|
||||
for topic in userdata.keys(): |
||||
client.subscribe(topic) |
||||
logging.info(f'Subscribing to topic: {topic}') |
||||
|
||||
|
||||
def _label_config_match(label_config, labels): |
||||
"""Action 'keep' and 'drop' in label_config: Matches joined 'source_labels' to 'regex'""" |
||||
source = label_config['separator'].join( |
||||
[labels[x] for x in label_config['source_labels']]) |
||||
logging.debug(f'_label_config_match source: {source}') |
||||
match = re.match(label_config['regex'], source) |
||||
|
||||
if label_config['action'] == 'keep': |
||||
logging.debug( |
||||
f"_label_config_match Action: {label_config['action']}, Keep msg: {match is not None}") |
||||
return match is not None |
||||
if label_config['action'] == 'drop': |
||||
logging.debug( |
||||
f"_label_config_match Action: {label_config['action']}, Drop msg: {match is not None}") |
||||
return match is None |
||||
else: |
||||
logging.debug( |
||||
f"_label_config_match Action: {label_config['action']} is not supported, metric is dropped") |
||||
return False |
||||
|
||||
|
||||
def _apply_label_config(labels, label_configs): |
||||
"""Create/change labels based on label_config in config file.""" |
||||
|
||||
for label_config in label_configs: |
||||
if label_config['action'] == 'replace': |
||||
_label_config_rename(label_config, labels) |
||||
else: |
||||
if not _label_config_match(label_config, labels): |
||||
return False |
||||
return True |
||||
|
||||
|
||||
def _label_config_rename(label_config, labels): |
||||
"""Action 'rename' in label_config: Add/change value for label 'target_label'""" |
||||
source = label_config['separator'].join( |
||||
[labels[x] for x in label_config['source_labels']]) |
||||
|
||||
if re.match(re.compile(label_config['regex']), source): |
||||
logging.debug(f'_label_config_rename source: {source}') |
||||
result = re.sub(label_config['regex'], |
||||
label_config['replacement'], source) |
||||
logging.debug(f'_label_config_rename result: {result}') |
||||
labels[label_config['target_label']] = result |
||||
|
||||
def convert_victron_value(victron_value): |
||||
"""Convert value from Victron Cerbo GX (which is a JSON string) to an actual numeric value. The format from Victron MQTT is JSON: {"value": 0} """ |
||||
victron_value_dict = json.loads(victron_value) |
||||
return victron_value_dict['value'] |
||||
|
||||
|
||||
def finalize_labels(labels): |
||||
"""Keep '__value__', and '__topic__' but remove all other labels starting with '__'""" |
||||
#labels['value'] = labels['__value__'] |
||||
labels['value'] = convert_victron_value(labels['__value__']) |
||||
labels['topic'] = labels['__topic__'] |
||||
|
||||
return {k: v for k, v in labels.items() if not k.startswith('__')} |
||||
|
||||
|
||||
def _update_metrics(metrics, msg): |
||||
"""For each metric on this topic, apply label renaming if present, and export to prometheus""" |
||||
for metric in metrics: |
||||
labels = {'__topic__': metric['topic'], |
||||
'__msg_topic__': msg.topic, '__value__': str(msg.payload, 'utf-8')} |
||||
|
||||
if 'label_configs' in metric: |
||||
# If action 'keep' in label_configs fails, or 'drop' succeeds, the metric is not updated |
||||
if not _apply_label_config(labels, metric['label_configs']): |
||||
continue |
||||
|
||||
# try to convert to float, but leave as is if conversion not possible |
||||
try: |
||||
labels['__value__'] = float(labels['__value__'].replace(',', '.')) |
||||
except ValueError: |
||||
logging.debug(f"Conversion of {labels['__value__']} to float not possible, continue with value as is.") |
||||
|
||||
logging.debug('_update_metrics all labels:') |
||||
logging.debug(labels) |
||||
|
||||
labels = finalize_labels(labels) |
||||
|
||||
derived_metric = metric.setdefault('derived_metric', |
||||
# Add derived metric for when the message was last received (timestamp in milliseconds) |
||||
{ |
||||
'name': f"{metric['name']}_last_received", |
||||
'help': f"Last received message for '{metric['name']}'", |
||||
'type': 'gauge' |
||||
} |
||||
) |
||||
derived_labels = {'topic': metric['topic'], |
||||
'value': int(round(time.time() * 1000))} |
||||
|
||||
_export_to_prometheus(metric['name'], metric, labels) |
||||
|
||||
_export_to_prometheus( |
||||
derived_metric['name'], derived_metric, derived_labels) |
||||
|
||||
|
||||
# noinspection PyUnusedLocal |
||||
def _on_message(client, userdata, msg): |
||||
"""The callback for when a PUBLISH message is received from the server.""" |
||||
logging.debug( |
||||
f'_on_message Msg received on topic: {msg.topic}, Value: {str(msg.payload)}') |
||||
|
||||
for topic in userdata.keys(): |
||||
if _topic_matches(topic, msg.topic): |
||||
_update_metrics(userdata[topic], msg) |
||||
|
||||
def victron_mqtt_keep_alive(mqtt_client): |
||||
""" Send keepalive to Cerbo GX """ |
||||
properties=Properties(PacketTypes.PUBLISH) |
||||
mqtt_client.publish('R/c0619ab1cab9/system/0/Serial','',2,properties=properties); |
||||
|
||||
def _mqtt_init(mqtt_config, metrics): |
||||
"""Setup mqtt connection""" |
||||
mqtt_client = mqtt.Client(userdata=metrics) |
||||
mqtt_client.on_connect = _on_connect |
||||
mqtt_client.on_message = _on_message |
||||
|
||||
if 'auth' in mqtt_config: |
||||
auth = _strip_config(mqtt_config['auth'], ['username', 'password']) |
||||
mqtt_client.username_pw_set(**auth) |
||||
|
||||
if 'tls' in mqtt_config: |
||||
tls_config = _strip_config(mqtt_config['tls'], [ |
||||
'ca_certs', 'certfile', 'keyfile', 'cert_reqs', 'tls_version']) |
||||
mqtt_client.tls_set(**tls_config) |
||||
|
||||
try: |
||||
mqtt_client.connect(**_strip_config(mqtt_config, |
||||
['host', 'port', 'keepalive'])) |
||||
except ConnectionRefusedError as err: |
||||
logging.critical( |
||||
f"Error connecting to {mqtt_config['host']}:{mqtt_config['port']}: {err.strerror}") |
||||
sys.exit(1) |
||||
|
||||
return mqtt_client |
||||
|
||||
|
||||
def _export_to_prometheus(name, metric, labels): |
||||
"""Export metric and labels to prometheus.""" |
||||
metric_wrappers = {'gauge': GaugeWrapper, |
||||
'counter': CounterWrapper, |
||||
'counter_absolute': CounterAbsoluteWrapper, |
||||
'summary': SummaryWrapper, |
||||
'histogram': HistogramWrapper, |
||||
'enum': EnumWrapper, |
||||
} |
||||
valid_types = metric_wrappers.keys() |
||||
if metric['type'] not in valid_types: |
||||
logging.error( |
||||
f"Metric type: {metric['type']}, is not a valid metric type. Must be one of: {valid_types} - ingnoring" |
||||
) |
||||
return |
||||
|
||||
value = labels['value'] |
||||
del labels['value'] |
||||
|
||||
sorted_labels = _get_sorted_tuple_list(labels) |
||||
label_names, label_values = list(zip(*sorted_labels)) |
||||
|
||||
prometheus_metric = None |
||||
if not metric.get('prometheus_metric') or not metric['prometheus_metric'].get('parent'): |
||||
# parent metric not seen before, create metric |
||||
additional_parameters = metric.get('parameters', {}) |
||||
|
||||
metric_wrapper = metric_wrappers[metric['type']] |
||||
prometheus_metric = metric_wrapper( |
||||
metric['name'], metric['help'], label_names, **additional_parameters) |
||||
metric['prometheus_metric'] = {} |
||||
metric['prometheus_metric']['parent'] = prometheus_metric |
||||
else: |
||||
prometheus_metric = metric['prometheus_metric']['parent'] |
||||
try: |
||||
prometheus_metric.update(label_values, value) |
||||
except ValueError as ve: |
||||
logging.error(f"Value {value} is not compatible with metric {metric['name']} of type {metric['type']}") |
||||
logging.exception('ve:') |
||||
|
||||
logging.debug( |
||||
f"_export_to_prometheus metric ({metric['type']}): {name}{labels} updated with value: {value}") |
||||
if logging.DEBUG >= logging.root.level: # log test data only in debugging mode |
||||
_log_test_data(metric, labels['topic'], value) |
||||
|
||||
|
||||
def _log_test_data(metric, topic, value): |
||||
try: |
||||
base_metric = metric['prometheus_metric']['parent'].metric.collect() |
||||
samples = {} |
||||
for child_metric in base_metric: |
||||
if child_metric.name.endswith('_last_received'): |
||||
# ignore derived metrics '*._last_received' |
||||
continue |
||||
first_sample = child_metric.samples[0] |
||||
for first_sample in child_metric.samples: |
||||
if first_sample.labels.get('topic', '') == topic: |
||||
samples[first_sample.name] = first_sample |
||||
|
||||
if len(samples) == 1: |
||||
logging.debug( |
||||
f"TEST_DATA: {topic}; {value}; {child_metric.name}; {json.dumps(first_sample.labels)}; {first_sample.value}; 0; True") |
||||
else: |
||||
out_value = {} |
||||
labels = first_sample.labels |
||||
for sample_name, first_sample in samples.items(): |
||||
suffix = sample_name[len(child_metric.name):] |
||||
out_value[suffix] = first_sample.value |
||||
if suffix == "_bucket": # buckets have extra "le" label |
||||
labels = first_sample.labels |
||||
logging.debug( |
||||
f"TEST_DATA: {topic}; {value}; {child_metric.name}; {json.dumps(labels)}; {json.dumps(out_value)}; 0; True") |
||||
except: # pylint: disable=bare-except |
||||
logging.exception("Failed to log TEST_DATA. ignoring.") |
||||
|
||||
|
||||
class GaugeWrapper(): |
||||
""" |
||||
Wrapper to provide generic interface to Gauge metric |
||||
""" |
||||
def __init__(self, name, help_text, label_names, *args, **kwargs) -> None: |
||||
self.metric = prometheus.Gauge( |
||||
name, help_text, list(label_names) |
||||
) |
||||
def update(self, label_values, value): |
||||
child = self.metric.labels(*label_values) |
||||
child.set(value) |
||||
return child |
||||
|
||||
|
||||
class CounterWrapper(): |
||||
""" |
||||
Wrapper to provide generic interface to Counter metric |
||||
""" |
||||
|
||||
def __init__(self, name, help_text, label_names, *args, **kwargs) -> None: |
||||
self.metric = prometheus.Counter( |
||||
name, help_text, list(label_names) |
||||
) |
||||
|
||||
def update(self, label_values, value): |
||||
child = self.metric.labels(*label_values) |
||||
child.inc(value) |
||||
return child |
||||
|
||||
class HistogramWrapper(): |
||||
""" |
||||
Wrapper to provide generic interface to Summary metric |
||||
""" |
||||
|
||||
class CounterAbsoluteWrapper(): |
||||
""" |
||||
Wrapper to provide generic interface to CounterAbsolute metric |
||||
""" |
||||
|
||||
def __init__(self, name, help_text, label_names, *args, **kwargs) -> None: |
||||
self.metric = utils.prometheus_additions.CounterAbsolute( |
||||
name, help_text, list(label_names) |
||||
) |
||||
|
||||
def update(self, label_values, value): |
||||
child = self.metric.labels(*label_values) |
||||
child.set(value) |
||||
return child |
||||
|
||||
|
||||
class SummaryWrapper(): |
||||
""" |
||||
Wrapper to provide generic interface to Summary metric |
||||
""" |
||||
|
||||
def __init__(self, name, help_text, label_names, *args, **kwargs) -> None: |
||||
self.metric = prometheus.Summary( |
||||
name, help_text, list(label_names) |
||||
) |
||||
|
||||
def update(self, label_values, value): |
||||
child = self.metric.labels(*label_values) |
||||
child.observe(value) |
||||
return child |
||||
|
||||
|
||||
class HistogramWrapper(): |
||||
""" |
||||
Wrapper to provide generic interface to Summary metric |
||||
""" |
||||
|
||||
def __init__(self, name, help_text, label_names, *args, **kwargs) -> None: |
||||
params = {} |
||||
if kwargs.get('buckets'): |
||||
if isinstance(kwargs['buckets'], str): |
||||
params['buckets'] = kwargs['buckets'].split(',') |
||||
else: |
||||
params['buckets'] = kwargs['buckets'] |
||||
|
||||
self.metric = prometheus.Histogram( |
||||
name, help_text, list(label_names), **params |
||||
) |
||||
|
||||
def update(self, label_values, value): |
||||
child = self.metric.labels(*label_values) |
||||
child.observe(value) |
||||
return child |
||||
|
||||
|
||||
class EnumWrapper(): |
||||
def __init__(self, name, help_text, label_names, *args, **kwargs) -> None: |
||||
params = {} |
||||
if kwargs.get('states'): |
||||
params['states'] = kwargs['states'] |
||||
|
||||
self.metric = prometheus.Enum( |
||||
name, help_text, list(label_names), **params |
||||
) |
||||
|
||||
def update(self, label_values, value): |
||||
child = self.metric.labels(*label_values) |
||||
child.state(value) |
||||
return child |
||||
|
||||
|
||||
def add_static_metric(timestamp): |
||||
g = prometheus.Gauge('mqtt_exporter_timestamp', 'Startup time of exporter in millis since EPOC (static)', |
||||
['exporter_version']) |
||||
g.labels(VERSION).set(timestamp) |
||||
|
||||
|
||||
def _get_sorted_tuple_list(source): |
||||
"""Return a sorted list of tuples""" |
||||
filtered_source = source.copy() |
||||
sorted_tuple_list = sorted( |
||||
list(filtered_source.items()), key=operator.itemgetter(0)) |
||||
return sorted_tuple_list |
||||
|
||||
|
||||
def _signal_handler(sig, frame): |
||||
# pylint: disable=E1101 |
||||
logging.info('Received {0}'.format(signal.Signals(sig).name)) |
||||
sys.exit(0) |
||||
|
||||
|
||||
def main(): |
||||
add_static_metric(int(time.time() * 1000)) |
||||
# Setup argument parsing |
||||
parser = argparse.ArgumentParser( |
||||
description='Simple program to export formatted MQTT messages to Prometheus') |
||||
parser.add_argument('-c', '--config', action='store', dest='config', default='conf', |
||||
help='Set config location (file or directory), default: \'conf\'') |
||||
options = parser.parse_args() |
||||
|
||||
# Initial logging to console |
||||
_log_setup({'logfile': '', 'level': 'info'}) |
||||
signal.signal(signal.SIGTERM, _signal_handler) |
||||
signal.signal(signal.SIGINT, _signal_handler) |
||||
|
||||
# Read config file from disk |
||||
from_file = _read_config(options.config) |
||||
config = _parse_config_and_add_defaults(from_file) |
||||
|
||||
# Set up logging |
||||
_log_setup(config['logging']) |
||||
|
||||
# Start prometheus exporter |
||||
logging.info( |
||||
f"Starting Prometheus exporter on port: {str(config['prometheus']['exporter_port'])}") |
||||
try: |
||||
prometheus.start_http_server(config['prometheus']['exporter_port']) |
||||
except OSError as err: |
||||
logging.critical( |
||||
f"Error starting Prometheus exporter: {err.strerror}") |
||||
sys.exit(1) |
||||
|
||||
# Set up mqtt client and loop forever |
||||
mqtt_client = _mqtt_init(config['mqtt'], config['metrics']) |
||||
|
||||
# loop_forever is a blocking call |
||||
#mqtt_client.loop_forever() |
||||
# loop_start re-schedules itself over and aover as a thread: place outside the main loop! |
||||
mqtt_client.loop_start() |
||||
|
||||
# Wake up Victron Energy Cerbo GX |
||||
victron_mqtt_keep_alive(mqtt_client) |
||||
|
||||
# Get system time for keepalive delay loop |
||||
starttime = time.time() |
||||
|
||||
while 1 == 1: |
||||
|
||||
# Should be called every 30 seconds to keep Victron Energy Cebo GX awake (keepalive) |
||||
if time.time() - starttime > 30: |
||||
#logging.info(f"Keepalive") |
||||
victron_mqtt_keep_alive(mqtt_client) |
||||
starttime = time.time() |
||||
|
||||
loop_stop() |
||||
|
||||
if __name__ == '__main__': |
||||
main() |
||||
Loading…
Reference in new issue