PE1RXF telemetry added

CHANGELOG.md

@@ -48,3 +48,10 @@ All notable changes to this project will be documented in this file.
 ### Changed
 
 - Reading weather station is now a scheduled task.
+
+## [0.1.3] - 2024-03-14
+
+### Added
+
+- PE1RXF telemetry to MQTT bridge
+- Forwarding of APRS messages to MQTT

README.md

@@ -2,6 +2,10 @@
 
 A basic Linux APRS iGate and APRS weather station with additional (optional) PE1RXF telemetry support.
 
+TODO:
+
+- change heater algoritm back to 5 min on/15 min off, but start at 92% in stead of 96%. If the cooling period is less than 15 minutes, the temperture readings are periodically off by 1-1.5 degrees.
+
 ## Requirements
 
 - Python3

aprs_telemetry_to_mqtt.py

@@ -0,0 +1,233 @@
+#!/usr/bin/python3
+''''
+# A bridge between PE1RXF APRS telemetry messaging and MQTT.
+# It uses pythonax25 (https://github.com/josefmtd/python-ax25)
+#
+# This program reads the registers of the PE1RXF weather station via ModBus RTU and sends it as
+# an APRS WX report over APRS. Additionally, it sends beacons and forwards received APRS messages
+# to the APRS-IS network. All configurable via a YAML file called pe1rxf_aprs.yml.
+#
+# This program also has a PE1RXF APRS telemetry to MQTT bridge, which is configurable via pe1rxf_telemetry.yml
+#
+# Copyright (C) 2023, 2024 M.T. Konstapel https://meezenest.nl/mees
+#
+# This file is part of weather_station
+#
+#    weather_station is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation, either version 3 of the License, or
+#    (at your option) any later version.
+#
+#    weather_station is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with weather_station. If not, see <https://www.gnu.org/licenses/>.
+'''
+import sys
+import random
+import time
+from time import gmtime, strftime
+#import os
+#from pathlib import Path
+import yaml
+from yaml.loader import SafeLoader
+#from paho.mqtt import client as mqtt_client
+import paho.mqtt.publish as publish
+import logging
+import json
+#import csv
+
+logger = logging.getLogger("aprs_telemetry_to_mqtt")
+
+class aprs_telemetry_to_mqtt:
+
+    # initiate class: define name configuration files
+    def __init__(self, telemetry_config_file):
+        self.config_file = telemetry_config_file
+        logger.info("Initializing telemetry to mqtt bridge.")
+
+    def read_settings(self):
+        if self.read_config_file() == 0:
+            return 0
+
+        if self.test_telemetry_settings() == 0:
+            return 0
+        return 1
+
+
+    def read_config_file (self):
+        try:
+            with open(self.config_file) as f:
+                self.config_file_settings = yaml.load(f, Loader=SafeLoader)
+        except:
+            print ("Telemetry configuration file ./" + self.config_file + " not found or syntax error in file.")
+            return 0
+        else:
+            return 1
+
+    def test_telemetry_settings(self):
+        try:
+            tmp = self.config_file_settings['global']['broker']
+            tmp = self.config_file_settings['global']['port']
+            tmp = self.config_file_settings['global']['topic_root']
+            tmp = self.config_file_settings['global']['publish_messages']
+            tmp = self.config_file_settings['global']['call']
+            tmp = self.config_file_settings['global']['weather_report_interval']
+            tmp = self.config_file_settings['global']['blacklist']
+            tmp = self.config_file_settings['topics']
+        except:
+            print ("Error in the telemetry configuration file.")
+            return 0
+        else:
+            #print (self.config_file_settings['global']['topic_root'])
+            tmp = self.config_file_settings['global']['topic_root']
+            #mqtt_client_id = f'{self.config_file_settings['global']['topic_root']}-{random.randint(0, 1000)}'
+            self.mqtt_client_id = f'{tmp}-{random.randint(0, 1000)}'
+            return 1
+
+    # Publish a single message to the MQTT broker
+    def publish(self, topic, message):
+        try:
+            publish.single(topic, message, hostname=self.config_file_settings['global']['broker'])
+        except:
+            logger.debug("Failed to connect to MQTT  broker.")
+        else:
+            logger.debug('Published: ' + topic + '=' + message)
+
+    # Checks if payload (APRS message) contains valid PE1RXF telemetry data.
+    # If so, it sends the formatted data to the MQTT broker.
+    # Configuration is done via the file pe1rxf_telemetry.yml
+    def publish_telemetry_message(self, source, ax_device, ax_address, payload):
+
+        values=0
+
+        for topics in self.config_file_settings['topics']:
+
+            # Check source call
+            if source == topics['call']:
+
+                # Check ax_port
+                if topics['ax_port'] == 'all' or topics['ax_port'] == ax_device:
+                    #print('Call found in configuration file')
+
+                    # split payload at colon. If it is a valid reply, we should get three
+                    # substrings: the first in empty, the second with the call of the ax25
+                    # interface and the thirth with the status of the outputs
+                    split_message=payload.split(":")
+                    if len(split_message) == 3:
+                        #Remove spaces from destination call and test if message is for the server
+                        if split_message[1].replace(" ", "") == ax_address:
+                            print ('Received from: ' + source + ' Telemetry: ' + split_message[2])
+
+                            # The telemetry is available in split_message[2], but we have to check if it contains any valid data
+                            # Try to split into seperate values (values should be seperated by a comma)
+                            values=split_message[2].split(",")
+                            # Test al values: should be numbers and nothing else
+                            for field in values:
+                                if not self.is_float(field):
+                                    return 0
+                            # One anoying thing of the PE1RXF telemetry standard is that there is also a message containing the status of the output bits.
+                            # These messages are interpreted as valid telemetry data by this program. The message is send after a '06' command. This program
+                            # does not request this message, but another program might. So we have to filter these messages output
+                            if len(values[0]) == 5:
+                                allowed = '0' + '1'
+                                # Removes from the original string all the characters that are allowed, leaving us with a set containing either a) nothing, or b) the #offending characters from the string:'
+                                if not set(values[0]) - set(allowed):
+                                    print ("Probably digital status bits. Ignore.")
+                                    return 0
+
+                            # Check if number of telemtry values and number of descriptions in yml file are the same. If not make then the same by appending to the shorted list.
+                            nr_of_values = len(values)
+                            nr_of_descriptions = len(topics['description'])
+
+                            if nr_of_values > nr_of_descriptions:
+                                items_to_add = nr_of_values - nr_of_descriptions
+                                for x in range(items_to_add):
+                                    topics['description'].append('NotDefined')
+                                print('Added ' + str(items_to_add) + ' to descriptions')
+                            elif nr_of_values < nr_of_descriptions:
+                                items_to_add = nr_of_descriptions - nr_of_values
+                                for x in range(items_to_add):
+                                    values.append('0.0')
+                                print('Added ' + str(items_to_add) + ' to values')
+                            else:
+                                print('values and description are of equal length: good!')
+
+                            # Loop through descriptions and send values from telemtry file along with it
+                            ''''
+                            for index, descr in enumerate(topics['description'], start=0):
+                                current_topic = self.config_file_settings['global']['topic_root'] + '/' + topics['name'] + '/' + descr
+                                #self.publish(client,current_topic,values[index])
+                                try:
+                                    publish.single(current_topic, values[index], hostname=self.config_file_settings['global']['broker'])
+                                except:
+                                    print("Failed to connect to MQTT  broker.")
+                                else:
+                                    print('Published: ' + current_topic + '=' + values[index])
+                            '''
+                            # Loop through descriptions and send values from telemtry file along with it
+                            publish_list = []
+                            for index, descr in enumerate(topics['description'], start=0):
+                                current_topic = self.config_file_settings['global']['topic_root'] + '/' + topics['name'] + '/' + descr
+
+                                publish_list.append({"topic": current_topic, "payload": values[index]})
+
+                            try:
+                                publish.multiple(publish_list, hostname=self.config_file_settings['global']['broker'])
+                            except:
+                                logger.debug("Failed to connect to MQTT  broker.")
+                            else:
+                                logger.debug('Published telemetry to mqtt broker.')
+                                logger.debug(publish_list)
+
+
+        return values
+
+    # Checks if payload (APRS message) contains valid message to us.
+    # If so, it sends the formatted data to the MQTT broker.
+    # Configuration is done via the file pe1rxf_telemetry.yml
+    def publish_aprs_messages(self, source, ax_device, payload):
+
+        if self.config_file_settings['global']['publish_messages'] == 'YES':
+
+            mqtt_message = {}
+
+            # Loop through blacklist and check if source is not in it
+            for call in self.config_file_settings['global']['blacklist']:
+                if call == source:
+                    print ("Call " + call + " blacklisted. Message not send to mqtt broker.")
+                    return 0
+
+            # If we come to here the sender is not in the blacklist
+            # But is could well be an ordinary aprs packet, so let's check if it is a message.'
+            # Split payload at colon. If it is a valid message, we should get three
+            # substrings: the first in empty, the second with the call of the ax25
+            # interface and the thirth with the message itself
+            split_message=payload.split(":")
+            if len(split_message) == 3:
+                #Remove spaces from destination call
+                split_message[1] = split_message[1].replace(" ", "")
+                # check if call defined in the configuration file is part of the destination call. This way we can also catch all the prefixes
+                if self.config_file_settings['global']['call'] in split_message[1]:
+                    print ('Received from: ' + source + ' payload: ' + split_message[2])
+                    mqtt_message['from'] = source
+                    mqtt_message['to'] = split_message[1]
+                    mqtt_message['port'] = ax_device
+                    mqtt_message['time'] =  time.strftime("%Y-%m-%d %H:%M:%S", gmtime())
+                    mqtt_message['message'] = split_message[2]
+
+                    message = json.dumps(mqtt_message)
+                    topic = self.config_file_settings['global']['topic_root'] + '/aprs_message'
+                    self.publish(topic, message)
+
+    # Test if string is a number
+    def is_float(self, v):
+        try:
+            f=float(v)
+        except:
+            return False
+
+        return True

pe1rxf_aprs.py

@@ -1,3 +1,4 @@
+#!/usr/bin/python3
 '''
 # A basic APRS iGate and APRS weather station with additional (optional) PE1RXF telemetry support
 #
@@ -37,12 +38,13 @@ from threading import Thread
 
 from weather_station_modbus import WeatherStation
 from config_reader import config_reader
+from aprs_telemetry_to_mqtt import aprs_telemetry_to_mqtt
 
 main_config_file = "pe1rxf_aprs.yml"
 telemetry_config_file = "pe1rxf_telemetry.yml"
 rflog_file = ""
 # Make Weather data global so scheduled task can use it
-WxData = []
+WxData = {}
 APRSIS = []
 
 axport = []
@@ -365,6 +367,14 @@ def send_telemetry():
 
     send_ax25('PE1RXF-3', 'PE1RXF-13', "APZMDM", 0, message)
 
+def publish_weather_data(mqtt_client):
+
+    payload = ':' + 'PE1RXF-3 ' + ':' + str(WxData['Wind direction']) + ',' + str(WxData['Wind speed']) + ',' + str(WxData['Wind gust']) + ',' + str(WxData['Rain last hour']) + ',' + str(WxData['Rain last 24 hours']) + ',' + str(WxData['Temperature']) + ',' + str(WxData['Humidity']) + ',' + str(WxData['Pressure']) + ',' + str(WxData['Temp backup']) + ',' + str(WxData['Status bits'])
+
+    telemetry=mqtt_client.publish_telemetry_message("PE1RXF-13", "ax1", "PE1RXF-3", payload)
+
+    return 1
+
 def read_weather_station(weather_station):
     global WxData
     #print ("Reading registers of weather station.")
@@ -372,6 +382,7 @@ def read_weather_station(weather_station):
         print ('No response from ModBus, even after 5 retries. Keep trying.')
     else:
         WxData = weather_station.wx_data
+        print (WxData)
 
 def check_heater(weather_station):
     # Check if heater is off, if so, turn it on
@@ -388,6 +399,19 @@ def run():
 
     global WxData
 
+    # Fill WxData with some sensible data to start:
+    WxData['ID'] = 0.0
+    WxData['Wind direction'] = 0.0
+    WxData['Wind speed'] = 0.0
+    WxData['Wind gust'] = 0.0
+    WxData['Rain last hour'] = 0.0
+    WxData['Rain last 24 hours'] = 0.0
+    WxData['Temperature'] = 0.0
+    WxData['Humidity'] = 0.0
+    WxData['Pressure'] = 0.0
+    WxData['Temp backup'] = 0.0
+    WxData['Status bits'] = 0.0
+
     # Debug logging for aprslib
     logging.basicConfig(level=logging.DEBUG) # level=10
 
@@ -405,6 +429,10 @@ def run():
     print ("Write APRS frames to: " + rflog_file)
     print ("Read configuration files.")
 
+    # Setup MQTT connection
+    mqtt_connection = aprs_telemetry_to_mqtt(telemetry_config_file)
+    mqtt_connection.read_settings()
+
     rx_socket = setup_ax25()
 
     # a valid passcode for the callsign is required in order to send
@@ -450,10 +478,6 @@ def run():
             interval = interval * 60    # from minutes to seconds
             schedule.every(interval - 59).to(interval + 59).seconds.do(send_aprs_beacon, entry)
 
-    # Schedule telemetry transmision
-    print("Scheduled telemetry transmission.")
-    schedule.every(10).minutes.do(send_telemetry)
-
     # Schedule check if heater is still on
     # So when the weather station is unplugged and plugged back in, the heater will be enabled again.
     schedule.every(10).minutes.do(check_heater, weather_station)
@@ -462,6 +486,15 @@ def run():
     print("Scheduled readout of weather station.")
     schedule.every(1).minutes.do(read_weather_station, weather_station)
 
+    # Schedule telemetry transmision
+    print("Scheduled telemetry transmission.")
+    schedule.every(10).minutes.do(send_telemetry)
+
+    print("Schedule mqtt weather publisher.")
+    interval = mqtt_connection.config_file_settings['global']['weather_report_interval']
+    if interval != 0:
+        schedule.every(interval).minutes.do(publish_weather_data, mqtt_connection)
+
     # Connect to incoming APRS-IS feed
     # by default `raw` is False, then each line is ran through aprslib.parse()
     # Set filter on incomming feed
@@ -516,11 +549,19 @@ def run():
 
                 #aprs_frame = axaddress[0] + '>' + destination +  ',' + ','.join(digipeaters) + ':' + payload
                 #print (aprs_frame)
+                if payload == 'NOT VALID':
+                    print (">>> Packet not valid, ignored.")
+                else:
                     log_ax25(axaddress[port], source, destination, ','.join(digipeaters), payload, 'R')
                     digipeaters.append('qAR')
                     digipeaters.append(Configuration.config_file_settings['aprsis']['call'])
                     send_aprsis(source, destination, ','.join(digipeaters), payload)
-                #telemetry=process_message(source, port, payload, client)
+
+                    # Check if APRS frame is PE1RXF telemetry
+                    telemetry=mqtt_connection.publish_telemetry_message(source, axdevice[port], axaddress[port], payload)
+
+                    # Check if APRS frame is a message to us
+                    mqtt_connection.publish_aprs_messages(source, axdevice[port], payload)
 
 
         #time.sleep(1) # Short sleep

pe1rxf_aprs.yml

@@ -7,25 +7,25 @@
 #
 
 # This section (ax25) is being depricated
-ax25:
+#ax25:
-    call: PE1RXF-13                   # Call from which transmissions are made
+#    call: PE1RXF-13                   # Call from which transmissions are made
-    destination: APZMDM               # APRS destination
+#    destination: APZMDM               # APRS destination
 
-    telemetry_port: ax1               # Linux AX.25 port to which telemetry is sent
+#    telemetry_port: ax1               # Linux AX.25 port to which telemetry is sent
-    telemetry_digi_path: 0            # Digipeater path for telemetry messages (0 = no path)
+#    telemetry_digi_path: 0            # Digipeater path for telemetry messages (0 = no path)
-    telemetry_interval: 5             # Time between telemetry transmissions
+#    telemetry_interval: 5             # Time between telemetry transmissions
-    telemetry_server: PE1RXF-3        # PE1RXF telemetry server call
+#    telemetry_server: PE1RXF-3        # PE1RXF telemetry server call
 
-    weather_report_port: ax0          # Linux AX.25 port to which telemetry is sent
+#    weather_report_port: ax0          # Linux AX.25 port to which telemetry is sent
-    weather_report_digi_path: WIDE2-2 # Digipeater path for weather reports (0 = no path)
+#    weather_report_digi_path: WIDE2-2 # Digipeater path for weather reports (0 = no path)
-    weather_report_interval: 10       # Time between weather report transmissions
+#    weather_report_interval: 10       # Time between weather report transmissions
 
 # Global settings
 global:
     log-rf: /home/marcel/test/pe1rxf_aprs-rf.log    # Log RF traffic to file (0=no logging)
 
 modbus:
-    port: /dev/serial/by-path/platform-3f980000.usb-usb-0:1.2:1.0-port0                # USB port to which RS-485 dongle is connected
+    port: /dev/serial/by-path/platform-3f980000.usb-usb-0:1.3:1.0-port0 # USB port to which RS-485 dongle is connected
     address: 14                         # ModBus address of weather station
 
 # APRS-IS section
@@ -50,12 +50,12 @@ weather:
       digi_path: WIDE2-2
       position: 5302.76N/00707.85E_
       interval: 10
-    - port: aprsis
+#    - port: aprsis
-      call: PE1RXF-13
+#      call: PE1RXF-13
-      destination: APZMDM
+#      destination: APZMDM
-      digi_path: 0
+#      digi_path: 0
-      position: 5302.76N/00707.85E_
+#      position: 5302.76N/00707.85E_
-      interval: 10
+#      interval: 10
 
 # APRS beacon section
 beacon:

pe1rxf_telemetry.yml

@@ -1,5 +1,61 @@
-# The settings for the PE1RXF telemetry server are configured in this file
+# The settings for the PE1RXF APRS telemetry to MQTT bridge.
-#
+# The program does some syntax checking, but not extensively. Be aware that the program may crash when there is an error in this file!
-# NOTE: At the start, the program randomizes the starting time of every individual periodic transmission,
-#       so even if all intervals are equal, the transmissions are not at the same time, but rather spread over time.
 #
+# Add APRS nodes under topics. More than one can be defined.
+
+# Global settings apply to all other entries
+global:
+        broker: mqtt.meezenest.nl           # The MQTT broker we are going to use
+        port: 1883                          # The tcp port of the MQTT broker
+        topic_root: hamnet_aprs_nodes       # MQTT topic root
+        weather_report_interval: 1          # Publish weather report from weather station on ModBus to MQTT broker every x minutes (0=do not publish)
+
+        publish_messages: YES               # The program can forward APRS messages addressed to us to the MQTT broker. If YES: publish APRS messages addressed to call to mqtt broker (/topic_root/aprs_message/).
+                                            # If NO: do not publish to mqtt broker
+        call: PE1RXF                        # Call used for APRS message publishing to mqtt (if no sufix is given, messages for all sufixes will be forwarded to the mqtt broker)
+        blacklist:                          # APRS messages from these calls are not published on the mqtt broker (for examle, place the calls from telemetry nodes here. These messages are processed via the 'topic' entry.
+            - PE1RXF-13                     # This way the messages are not also published as plain messages to the mqtt broker.
+            - PE1RXF-3
+            - PE1RXF-5
+            - PE1RXF-6
+            - PE1RXF-8
+            - PE1RXF-9
+topics:
+    # MQTT topic: each telemtry node has its own name (sub root) and must be unique
+    - name: solar_generator
+      # telemetry_file is obsolete. Use call instead.
+      #telemetry_file: /home/marcel/ham/aprs_utils/aprs_log/latest_telemetry_PE1RXF-9.dat
+      # Call of the telemetry node
+      call: PE1RXF-9
+      # AX.25 port to listen on (all for all ports)
+      ax_port: all
+      # Defines the names of the values in the telemetry data. These names are used to publish to the MQTT broker.
+      # Make sure the number of descriptions match the number of values in the telemetry data!
+      description:
+        - soc
+        - voltage
+        - power
+        - temperature
+    - name: wx_workshop
+      call: PE1RXF-6
+      ax_port: all
+      description:
+        - temperature
+        - humidity
+    # Definition of the build in weather station telemetry. Set interval in global/weather_report_interval
+    - name: weather_station
+      call: PE1RXF-13
+      ax_port: all
+      description:
+        - wind_direction
+        - wind_speed
+        - wind_gust
+        - rain_lasthour
+        - rain_24hours
+        - temperature
+        - humidity
+        - pressure
+        - temperature_backup
+        - status_bits
+
+