From 6a0b201507c2fc1a3978c97b0639d227f6eaf748 Mon Sep 17 00:00:00 2001 From: marcel Date: Thu, 14 Mar 2024 13:17:38 +0100 Subject: [PATCH] PE1RXF telemetry added --- CHANGELOG.md | 7 ++ README.md | 4 + aprs_telemetry_to_mqtt.py | 233 ++++++++++++++++++++++++++++++++++++++ pe1rxf_aprs.py | 61 ++++++++-- pe1rxf_aprs.yml | 34 +++--- pe1rxf_telemetry.yml | 64 ++++++++++- 6 files changed, 372 insertions(+), 31 deletions(-) create mode 100755 aprs_telemetry_to_mqtt.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 075be62..6345738 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -48,3 +48,10 @@ All notable changes to this project will be documented in this file. ### Changed - Reading weather station is now a scheduled task. + +## [0.1.3] - 2024-03-14 + +### Added + +- PE1RXF telemetry to MQTT bridge +- Forwarding of APRS messages to MQTT diff --git a/README.md b/README.md index 2b6cd94..48e5e83 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,10 @@ A basic Linux APRS iGate and APRS weather station with additional (optional) PE1RXF telemetry support. +TODO: + +- change heater algoritm back to 5 min on/15 min off, but start at 92% in stead of 96%. If the cooling period is less than 15 minutes, the temperture readings are periodically off by 1-1.5 degrees. + ## Requirements - Python3 diff --git a/aprs_telemetry_to_mqtt.py b/aprs_telemetry_to_mqtt.py new file mode 100755 index 0000000..863c91c --- /dev/null +++ b/aprs_telemetry_to_mqtt.py @@ -0,0 +1,233 @@ +#!/usr/bin/python3 +'''' +# A bridge between PE1RXF APRS telemetry messaging and MQTT. +# It uses pythonax25 (https://github.com/josefmtd/python-ax25) +# +# This program reads the registers of the PE1RXF weather station via ModBus RTU and sends it as +# an APRS WX report over APRS. Additionally, it sends beacons and forwards received APRS messages +# to the APRS-IS network. All configurable via a YAML file called pe1rxf_aprs.yml. +# +# This program also has a PE1RXF APRS telemetry to MQTT bridge, which is configurable via pe1rxf_telemetry.yml +# +# Copyright (C) 2023, 2024 M.T. Konstapel https://meezenest.nl/mees +# +# This file is part of weather_station +# +# weather_station is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weather_station is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with weather_station. If not, see . +''' +import sys +import random +import time +from time import gmtime, strftime +#import os +#from pathlib import Path +import yaml +from yaml.loader import SafeLoader +#from paho.mqtt import client as mqtt_client +import paho.mqtt.publish as publish +import logging +import json +#import csv + +logger = logging.getLogger("aprs_telemetry_to_mqtt") + +class aprs_telemetry_to_mqtt: + + # initiate class: define name configuration files + def __init__(self, telemetry_config_file): + self.config_file = telemetry_config_file + logger.info("Initializing telemetry to mqtt bridge.") + + def read_settings(self): + if self.read_config_file() == 0: + return 0 + + if self.test_telemetry_settings() == 0: + return 0 + return 1 + + + def read_config_file (self): + try: + with open(self.config_file) as f: + self.config_file_settings = yaml.load(f, Loader=SafeLoader) + except: + print ("Telemetry configuration file ./" + self.config_file + " not found or syntax error in file.") + return 0 + else: + return 1 + + def test_telemetry_settings(self): + try: + tmp = self.config_file_settings['global']['broker'] + tmp = self.config_file_settings['global']['port'] + tmp = self.config_file_settings['global']['topic_root'] + tmp = self.config_file_settings['global']['publish_messages'] + tmp = self.config_file_settings['global']['call'] + tmp = self.config_file_settings['global']['weather_report_interval'] + tmp = self.config_file_settings['global']['blacklist'] + tmp = self.config_file_settings['topics'] + except: + print ("Error in the telemetry configuration file.") + return 0 + else: + #print (self.config_file_settings['global']['topic_root']) + tmp = self.config_file_settings['global']['topic_root'] + #mqtt_client_id = f'{self.config_file_settings['global']['topic_root']}-{random.randint(0, 1000)}' + self.mqtt_client_id = f'{tmp}-{random.randint(0, 1000)}' + return 1 + + # Publish a single message to the MQTT broker + def publish(self, topic, message): + try: + publish.single(topic, message, hostname=self.config_file_settings['global']['broker']) + except: + logger.debug("Failed to connect to MQTT broker.") + else: + logger.debug('Published: ' + topic + '=' + message) + + # Checks if payload (APRS message) contains valid PE1RXF telemetry data. + # If so, it sends the formatted data to the MQTT broker. + # Configuration is done via the file pe1rxf_telemetry.yml + def publish_telemetry_message(self, source, ax_device, ax_address, payload): + + values=0 + + for topics in self.config_file_settings['topics']: + + # Check source call + if source == topics['call']: + + # Check ax_port + if topics['ax_port'] == 'all' or topics['ax_port'] == ax_device: + #print('Call found in configuration file') + + # split payload at colon. If it is a valid reply, we should get three + # substrings: the first in empty, the second with the call of the ax25 + # interface and the thirth with the status of the outputs + split_message=payload.split(":") + if len(split_message) == 3: + #Remove spaces from destination call and test if message is for the server + if split_message[1].replace(" ", "") == ax_address: + print ('Received from: ' + source + ' Telemetry: ' + split_message[2]) + + # The telemetry is available in split_message[2], but we have to check if it contains any valid data + # Try to split into seperate values (values should be seperated by a comma) + values=split_message[2].split(",") + # Test al values: should be numbers and nothing else + for field in values: + if not self.is_float(field): + return 0 + # One anoying thing of the PE1RXF telemetry standard is that there is also a message containing the status of the output bits. + # These messages are interpreted as valid telemetry data by this program. The message is send after a '06' command. This program + # does not request this message, but another program might. So we have to filter these messages output + if len(values[0]) == 5: + allowed = '0' + '1' + # Removes from the original string all the characters that are allowed, leaving us with a set containing either a) nothing, or b) the #offending characters from the string:' + if not set(values[0]) - set(allowed): + print ("Probably digital status bits. Ignore.") + return 0 + + # Check if number of telemtry values and number of descriptions in yml file are the same. If not make then the same by appending to the shorted list. + nr_of_values = len(values) + nr_of_descriptions = len(topics['description']) + + if nr_of_values > nr_of_descriptions: + items_to_add = nr_of_values - nr_of_descriptions + for x in range(items_to_add): + topics['description'].append('NotDefined') + print('Added ' + str(items_to_add) + ' to descriptions') + elif nr_of_values < nr_of_descriptions: + items_to_add = nr_of_descriptions - nr_of_values + for x in range(items_to_add): + values.append('0.0') + print('Added ' + str(items_to_add) + ' to values') + else: + print('values and description are of equal length: good!') + + # Loop through descriptions and send values from telemtry file along with it + '''' + for index, descr in enumerate(topics['description'], start=0): + current_topic = self.config_file_settings['global']['topic_root'] + '/' + topics['name'] + '/' + descr + #self.publish(client,current_topic,values[index]) + try: + publish.single(current_topic, values[index], hostname=self.config_file_settings['global']['broker']) + except: + print("Failed to connect to MQTT broker.") + else: + print('Published: ' + current_topic + '=' + values[index]) + ''' + # Loop through descriptions and send values from telemtry file along with it + publish_list = [] + for index, descr in enumerate(topics['description'], start=0): + current_topic = self.config_file_settings['global']['topic_root'] + '/' + topics['name'] + '/' + descr + + publish_list.append({"topic": current_topic, "payload": values[index]}) + + try: + publish.multiple(publish_list, hostname=self.config_file_settings['global']['broker']) + except: + logger.debug("Failed to connect to MQTT broker.") + else: + logger.debug('Published telemetry to mqtt broker.') + logger.debug(publish_list) + + + return values + + # Checks if payload (APRS message) contains valid message to us. + # If so, it sends the formatted data to the MQTT broker. + # Configuration is done via the file pe1rxf_telemetry.yml + def publish_aprs_messages(self, source, ax_device, payload): + + if self.config_file_settings['global']['publish_messages'] == 'YES': + + mqtt_message = {} + + # Loop through blacklist and check if source is not in it + for call in self.config_file_settings['global']['blacklist']: + if call == source: + print ("Call " + call + " blacklisted. Message not send to mqtt broker.") + return 0 + + # If we come to here the sender is not in the blacklist + # But is could well be an ordinary aprs packet, so let's check if it is a message.' + # Split payload at colon. If it is a valid message, we should get three + # substrings: the first in empty, the second with the call of the ax25 + # interface and the thirth with the message itself + split_message=payload.split(":") + if len(split_message) == 3: + #Remove spaces from destination call + split_message[1] = split_message[1].replace(" ", "") + # check if call defined in the configuration file is part of the destination call. This way we can also catch all the prefixes + if self.config_file_settings['global']['call'] in split_message[1]: + print ('Received from: ' + source + ' payload: ' + split_message[2]) + mqtt_message['from'] = source + mqtt_message['to'] = split_message[1] + mqtt_message['port'] = ax_device + mqtt_message['time'] = time.strftime("%Y-%m-%d %H:%M:%S", gmtime()) + mqtt_message['message'] = split_message[2] + + message = json.dumps(mqtt_message) + topic = self.config_file_settings['global']['topic_root'] + '/aprs_message' + self.publish(topic, message) + + # Test if string is a number + def is_float(self, v): + try: + f=float(v) + except: + return False + + return True diff --git a/pe1rxf_aprs.py b/pe1rxf_aprs.py index 87fff2d..085e1bf 100644 --- a/pe1rxf_aprs.py +++ b/pe1rxf_aprs.py @@ -1,3 +1,4 @@ +#!/usr/bin/python3 ''' # A basic APRS iGate and APRS weather station with additional (optional) PE1RXF telemetry support # @@ -37,12 +38,13 @@ from threading import Thread from weather_station_modbus import WeatherStation from config_reader import config_reader +from aprs_telemetry_to_mqtt import aprs_telemetry_to_mqtt main_config_file = "pe1rxf_aprs.yml" telemetry_config_file = "pe1rxf_telemetry.yml" rflog_file = "" # Make Weather data global so scheduled task can use it -WxData = [] +WxData = {} APRSIS = [] axport = [] @@ -365,6 +367,14 @@ def send_telemetry(): send_ax25('PE1RXF-3', 'PE1RXF-13', "APZMDM", 0, message) +def publish_weather_data(mqtt_client): + + payload = ':' + 'PE1RXF-3 ' + ':' + str(WxData['Wind direction']) + ',' + str(WxData['Wind speed']) + ',' + str(WxData['Wind gust']) + ',' + str(WxData['Rain last hour']) + ',' + str(WxData['Rain last 24 hours']) + ',' + str(WxData['Temperature']) + ',' + str(WxData['Humidity']) + ',' + str(WxData['Pressure']) + ',' + str(WxData['Temp backup']) + ',' + str(WxData['Status bits']) + + telemetry=mqtt_client.publish_telemetry_message("PE1RXF-13", "ax1", "PE1RXF-3", payload) + + return 1 + def read_weather_station(weather_station): global WxData #print ("Reading registers of weather station.") @@ -372,6 +382,7 @@ def read_weather_station(weather_station): print ('No response from ModBus, even after 5 retries. Keep trying.') else: WxData = weather_station.wx_data + print (WxData) def check_heater(weather_station): # Check if heater is off, if so, turn it on @@ -388,6 +399,19 @@ def run(): global WxData + # Fill WxData with some sensible data to start: + WxData['ID'] = 0.0 + WxData['Wind direction'] = 0.0 + WxData['Wind speed'] = 0.0 + WxData['Wind gust'] = 0.0 + WxData['Rain last hour'] = 0.0 + WxData['Rain last 24 hours'] = 0.0 + WxData['Temperature'] = 0.0 + WxData['Humidity'] = 0.0 + WxData['Pressure'] = 0.0 + WxData['Temp backup'] = 0.0 + WxData['Status bits'] = 0.0 + # Debug logging for aprslib logging.basicConfig(level=logging.DEBUG) # level=10 @@ -405,6 +429,10 @@ def run(): print ("Write APRS frames to: " + rflog_file) print ("Read configuration files.") + # Setup MQTT connection + mqtt_connection = aprs_telemetry_to_mqtt(telemetry_config_file) + mqtt_connection.read_settings() + rx_socket = setup_ax25() # a valid passcode for the callsign is required in order to send @@ -450,10 +478,6 @@ def run(): interval = interval * 60 # from minutes to seconds schedule.every(interval - 59).to(interval + 59).seconds.do(send_aprs_beacon, entry) - # Schedule telemetry transmision - print("Scheduled telemetry transmission.") - schedule.every(10).minutes.do(send_telemetry) - # Schedule check if heater is still on # So when the weather station is unplugged and plugged back in, the heater will be enabled again. schedule.every(10).minutes.do(check_heater, weather_station) @@ -462,6 +486,15 @@ def run(): print("Scheduled readout of weather station.") schedule.every(1).minutes.do(read_weather_station, weather_station) + # Schedule telemetry transmision + print("Scheduled telemetry transmission.") + schedule.every(10).minutes.do(send_telemetry) + + print("Schedule mqtt weather publisher.") + interval = mqtt_connection.config_file_settings['global']['weather_report_interval'] + if interval != 0: + schedule.every(interval).minutes.do(publish_weather_data, mqtt_connection) + # Connect to incoming APRS-IS feed # by default `raw` is False, then each line is ran through aprslib.parse() # Set filter on incomming feed @@ -516,11 +549,19 @@ def run(): #aprs_frame = axaddress[0] + '>' + destination + ',' + ','.join(digipeaters) + ':' + payload #print (aprs_frame) - log_ax25(axaddress[port], source, destination, ','.join(digipeaters), payload, 'R') - digipeaters.append('qAR') - digipeaters.append(Configuration.config_file_settings['aprsis']['call']) - send_aprsis(source, destination, ','.join(digipeaters), payload) - #telemetry=process_message(source, port, payload, client) + if payload == 'NOT VALID': + print (">>> Packet not valid, ignored.") + else: + log_ax25(axaddress[port], source, destination, ','.join(digipeaters), payload, 'R') + digipeaters.append('qAR') + digipeaters.append(Configuration.config_file_settings['aprsis']['call']) + send_aprsis(source, destination, ','.join(digipeaters), payload) + + # Check if APRS frame is PE1RXF telemetry + telemetry=mqtt_connection.publish_telemetry_message(source, axdevice[port], axaddress[port], payload) + + # Check if APRS frame is a message to us + mqtt_connection.publish_aprs_messages(source, axdevice[port], payload) #time.sleep(1) # Short sleep diff --git a/pe1rxf_aprs.yml b/pe1rxf_aprs.yml index c86ff98..a5ed8ce 100644 --- a/pe1rxf_aprs.yml +++ b/pe1rxf_aprs.yml @@ -7,25 +7,25 @@ # # This section (ax25) is being depricated -ax25: - call: PE1RXF-13 # Call from which transmissions are made - destination: APZMDM # APRS destination +#ax25: +# call: PE1RXF-13 # Call from which transmissions are made +# destination: APZMDM # APRS destination - telemetry_port: ax1 # Linux AX.25 port to which telemetry is sent - telemetry_digi_path: 0 # Digipeater path for telemetry messages (0 = no path) - telemetry_interval: 5 # Time between telemetry transmissions - telemetry_server: PE1RXF-3 # PE1RXF telemetry server call +# telemetry_port: ax1 # Linux AX.25 port to which telemetry is sent +# telemetry_digi_path: 0 # Digipeater path for telemetry messages (0 = no path) +# telemetry_interval: 5 # Time between telemetry transmissions +# telemetry_server: PE1RXF-3 # PE1RXF telemetry server call - weather_report_port: ax0 # Linux AX.25 port to which telemetry is sent - weather_report_digi_path: WIDE2-2 # Digipeater path for weather reports (0 = no path) - weather_report_interval: 10 # Time between weather report transmissions +# weather_report_port: ax0 # Linux AX.25 port to which telemetry is sent +# weather_report_digi_path: WIDE2-2 # Digipeater path for weather reports (0 = no path) +# weather_report_interval: 10 # Time between weather report transmissions # Global settings global: log-rf: /home/marcel/test/pe1rxf_aprs-rf.log # Log RF traffic to file (0=no logging) modbus: - port: /dev/serial/by-path/platform-3f980000.usb-usb-0:1.2:1.0-port0 # USB port to which RS-485 dongle is connected + port: /dev/serial/by-path/platform-3f980000.usb-usb-0:1.3:1.0-port0 # USB port to which RS-485 dongle is connected address: 14 # ModBus address of weather station # APRS-IS section @@ -50,12 +50,12 @@ weather: digi_path: WIDE2-2 position: 5302.76N/00707.85E_ interval: 10 - - port: aprsis - call: PE1RXF-13 - destination: APZMDM - digi_path: 0 - position: 5302.76N/00707.85E_ - interval: 10 +# - port: aprsis +# call: PE1RXF-13 +# destination: APZMDM +# digi_path: 0 +# position: 5302.76N/00707.85E_ +# interval: 10 # APRS beacon section beacon: diff --git a/pe1rxf_telemetry.yml b/pe1rxf_telemetry.yml index df2f28e..61b90ef 100644 --- a/pe1rxf_telemetry.yml +++ b/pe1rxf_telemetry.yml @@ -1,5 +1,61 @@ -# The settings for the PE1RXF telemetry server are configured in this file -# -# NOTE: At the start, the program randomizes the starting time of every individual periodic transmission, -# so even if all intervals are equal, the transmissions are not at the same time, but rather spread over time. +# The settings for the PE1RXF APRS telemetry to MQTT bridge. +# The program does some syntax checking, but not extensively. Be aware that the program may crash when there is an error in this file! # +# Add APRS nodes under topics. More than one can be defined. + +# Global settings apply to all other entries +global: + broker: mqtt.meezenest.nl # The MQTT broker we are going to use + port: 1883 # The tcp port of the MQTT broker + topic_root: hamnet_aprs_nodes # MQTT topic root + weather_report_interval: 1 # Publish weather report from weather station on ModBus to MQTT broker every x minutes (0=do not publish) + + publish_messages: YES # The program can forward APRS messages addressed to us to the MQTT broker. If YES: publish APRS messages addressed to call to mqtt broker (/topic_root/aprs_message/). + # If NO: do not publish to mqtt broker + call: PE1RXF # Call used for APRS message publishing to mqtt (if no sufix is given, messages for all sufixes will be forwarded to the mqtt broker) + blacklist: # APRS messages from these calls are not published on the mqtt broker (for examle, place the calls from telemetry nodes here. These messages are processed via the 'topic' entry. + - PE1RXF-13 # This way the messages are not also published as plain messages to the mqtt broker. + - PE1RXF-3 + - PE1RXF-5 + - PE1RXF-6 + - PE1RXF-8 + - PE1RXF-9 +topics: + # MQTT topic: each telemtry node has its own name (sub root) and must be unique + - name: solar_generator + # telemetry_file is obsolete. Use call instead. + #telemetry_file: /home/marcel/ham/aprs_utils/aprs_log/latest_telemetry_PE1RXF-9.dat + # Call of the telemetry node + call: PE1RXF-9 + # AX.25 port to listen on (all for all ports) + ax_port: all + # Defines the names of the values in the telemetry data. These names are used to publish to the MQTT broker. + # Make sure the number of descriptions match the number of values in the telemetry data! + description: + - soc + - voltage + - power + - temperature + - name: wx_workshop + call: PE1RXF-6 + ax_port: all + description: + - temperature + - humidity + # Definition of the build in weather station telemetry. Set interval in global/weather_report_interval + - name: weather_station + call: PE1RXF-13 + ax_port: all + description: + - wind_direction + - wind_speed + - wind_gust + - rain_lasthour + - rain_24hours + - temperature + - humidity + - pressure + - temperature_backup + - status_bits + +