Add files via upload

* Added v1.0.1

SondeHubUploader/SondeHubUploader.py

@@ -15,6 +15,7 @@ class SondeHubUploader:
 
     import SondeHubUploader.shuConfig as shuConfig
     import SondeHubUploader.logger as logger
+    import SondeHubUploader.crc as crc
     import SondeHubUploader.threads as threads
     import SondeHubUploader.handleData as handleData
     import SondeHubUploader.conversions as conversions
@@ -28,11 +29,12 @@ class SondeHubUploader:
     def __init__(self, args):
         # Save the provided configuration parameters
         self.__dict__.update(args)
-
-        # Define a logger object
-        self.loggerObj = logging.getLogger('logger')
-        # Configure the logger
-        self.logger.configure_logger(self, self.loggerObj, self.loglevelp, self.loglevelw, self.writel)
+
+        # Create a logger
+        self.logger.create_logger(self, self.loglevelp, self.loglevelw, self.writel)
+
+        # Create a crc calculator
+        self.crc.crc_create_calculator(self, 16, 0x1021, 0xFFFF, 0xFFFF, True, True)
 
         # Used to break out of while-loops when the SondeHubUploader is terminated
         self.running = True

SondeHubUploader/crc.py

@@ -0,0 +1,21 @@
+# crc.py - Functions for CRC
+#
+# Copyright (C) Simon Schäfer <[email protected]>
+#
+# Released under GNU GPL v3 or later
+
+
+import crc
+
+
+# Create a crc calculator
+def crc_create_calculator(self, width, polynomial, init_value, final_xor_value, reverse_input, reverse_output):
+    configuration = crc.Configuration(
+        width=width,
+        polynomial=polynomial,
+        init_value=init_value,
+        final_xor_value=final_xor_value,
+        reverse_input=reverse_input,
+        reverse_output=reverse_output,
+    )
+    self.crc_calculator = crc.Calculator(configuration)

SondeHubUploader/handleData.py

@@ -39,6 +39,11 @@ def parse_aprs_package(self, aprs_package):
         if start_index != -1:
             # For parsing, the end index of the value needs to be found as well
             end_index = aprs_package_string[start_index:].find(end_string) + start_index
+            # End index will be start index - 1 if it was not found
+            # In that case the telemetry parameter is the last one in the aprs package string
+            if end_index < start_index:
+                # The end index is therefore set to the last index in the aprs package string
+                end_index = len(aprs_package_string)-1
             # The actual parsing is again done using the parse function
             try:
                 telemetry[parameter] = parse_function(aprs_package_string[start_index:end_index])
@@ -55,6 +60,11 @@ def parse_aprs_package(self, aprs_package):
         start_index = self.utils.aprs_package_string_find_key(aprs_package_string, start_string)
         if start_index != -1:
             end_index = aprs_package_string[start_index:].find(end_string) + start_index
+            # End index will be start index - 1 if it was not found
+            # In that case the telemetry parameter is the last one in the aprs package string
+            if end_index < start_index:
+                # The end index is therefore set to the last index in the aprs package string
+                end_index = len(aprs_package_string) - 1
             # The actual parsing is different compared to the optional telemetry parameters
             # Optional multivalue telemetry parameters contain a list of subparameters
             # The subparameters are parsed using the subparameter parse function
@@ -178,6 +188,10 @@ def reformat_telemetry(self, telemetry):
                 # For M10 radiosondes, the serial provided by dxlAPRS is missing some dashes
                 if reformatted_telemetry['type'] == 'M10':
                     reformatted_telemetry['serial'] = reformatted_telemetry['serial'][0:3] + '-' + reformatted_telemetry['serial'][3] + '-' + reformatted_telemetry['serial'][4:]
+                # For M20 radiosondes, the serial might have some sort of number in square brackets attached
+                # This needs to me removed
+                if reformatted_telemetry['type'] == 'M20':
+                    reformatted_telemetry['serial'] = reformatted_telemetry['serial'].split('[', 1)[0]
             # The datetime is composed of the time provided by dxlAPRS and a date that is added manually
             reformatted_telemetry['datetime'] = self.utils.fix_datetime(self, telemetry['hour'], telemetry['minute'], telemetry['second'], True if ref_datetime == 'GPS' else False).strftime('%Y-%m-%dT%H:%M:%S.%fZ')
             # For most radiosondes, the framenumber can be taken directly from the telemetry

SondeHubUploader/logger.py

@@ -1,4 +1,4 @@
-# logging.py - Functions for logger configuration
+# logging.py - Functions for logger creation
 #
 # Copyright (C) Simon Schäfer <[email protected]>
 #
@@ -10,7 +10,10 @@
 
 
 # Configure logger
-def configure_logger(self, logger, loglevelp, loglevelw, savel):
+def create_logger(self, loglevelp, loglevelw, savel):
+    # Define a logger
+    self.loggerObj = logging.getLogger('logger')
+
     # Add a custom logging level for detailed debugging
     add_logging_level('DEBUG_DETAIL', self.shuConfig.loglevel[5])
 
@@ -19,16 +22,16 @@ def configure_logger(self, logger, loglevelp, loglevelw, savel):
     c_handler = logging.StreamHandler()
     c_handler.setFormatter(c_format)
     c_handler.setLevel(self.shuConfig.loglevel[loglevelp])
-    logger.addHandler(c_handler)
+    self.loggerObj.addHandler(c_handler)
     # A optional 'FileHandler' might be added to the logger
     if savel:
         f_format = logging.Formatter('[%(asctime)s - %(levelname)s] %(message)s')
         f_handler = logging.FileHandler(self.filepath + '/' + 'log.log')
         f_handler.setFormatter(f_format)
         f_handler.setLevel(self.shuConfig.loglevel[loglevelw])
-        logger.addHandler(f_handler)
+        self.loggerObj.addHandler(f_handler)
     # A level must also be set for the logger itself, not only for the handlers
-    logger.setLevel(self.shuConfig.loglevel[5])
+    self.loggerObj.setLevel(self.shuConfig.loglevel[5])
 
 
 # Add a custom logging level

SondeHubUploader/shuConfig.py

@@ -26,10 +26,9 @@
 sondehub_telemetry_url = 'https://api.v2.sondehub.org/sondes/telemetry'
 sondehub_station_url = 'https://api.v2.sondehub.org/listeners'
 
-
 # Software definitions
 software_name = 'dxlAPRS-SHUE'
-software_version = '1.0.0'
+software_version = '1.0.1'
 
 # Status code definitions
 status_code_ok = 200
@@ -120,7 +119,7 @@
     'control': [lambda a: True if a == hex(0x3) else False, False, False, None, None],
     'protocol_id': [lambda a: True if a == hex(0xF0) else False, False, False, None, None],
     'data_type': [lambda a: True if a == ';' else False, False, False, None, None],
-    'serial': [None, ['RS41', 'RS92', 'DFM'], False, None, None],
+    'serial': [lambda a: True if len(a) >= 4 else False, ['RS41', 'RS92', 'DFM'], False, None, None],
     'hour': [lambda a: True if a <= 23 else False, True, False, None, None],
     'minute': [lambda a: True if a <= 59 else False, True, False, None, None],
     'second': [lambda a: True if a <= 59 else False, True, False, None, None],
@@ -156,7 +155,7 @@
     'elevation': [lambda a: True if 0 <= a <= 90 else False, False, False, None, None],
     'dist': [lambda a: True if 0 <= a <= 1500 else False, False, False, None, None],
     'dev': [None, False, False, None, None],
-    'ser': [None, ['M10', 'M20', 'MRZ', 'MEISEI'], False, None, None],
+    'ser': [lambda a: True if len(a) >= 4 else False, ['M10', 'M20', 'MRZ', 'MEISEI'], False, None, None],
     'tx_past_burst_hour': [lambda a: True if a <= 23 else False, False, False, None, None],
     'tx_past_burst_minute': [lambda a: True if a <= 59 else False, False, False, None, None],
     'tx_past_burst_second': [lambda a: True if a <= 59 else False, False, False, None, None],
@@ -252,8 +251,8 @@
     'RS92': ['Vaisala', 'RS92', None, ['serial', 0], 'fn', 5, 'GPS', 'GPS'],
     'DFM': ['Graw', 'DFM', ['DFM06', 'DFM09', 'DFM09P', 'DFM17'], ['serial', 1], 'gps', 2, 'UTC', 'GPS'],
     'iMET': ['Intermet Systems', 'iMet-4', None, 'IMET', 'fn', 0, 'GPS', 'MSL'],
-    #'M10': ['Meteomodem', 'M10', None, ['ser', 0], 'gpsleap', 2, 'UTC', 'GPS'],
-    #'M20': ['Meteomodem', 'M20', None, ['ser', 0], 'gps', 2, 'GPS', 'GPS'],
+    'M10': ['Meteomodem', 'M10', None, ['ser', 0], 'gpsleap', 2, 'UTC', 'GPS'],
+    'M20': ['Meteomodem', 'M20', None, ['ser', 0], 'gps', 2, 'GPS', 'GPS'],
     'MRZ': ['Meteo-Radiy', 'MRZ', None, ['ser', 0], 'gps', 5, 'UTC', 'GPS'],
     'MEISEI': ['Meisei', 'IMS100', None, ['ser', 7], 'fn', 1, 'UTC', 'GPS']
 }

SondeHubUploader/threads.py

@@ -44,36 +44,40 @@ def process_aprs_queue(self):
             # Optionally write the raw data
             if self.writeo:
                 self.writeData.write_raw_data(self, aprs_package)
-            # Parse the APRS package
-            telemetry = self.handleData.parse_aprs_package(self, aprs_package)
-            self.loggerObj.debug('APRS package parsed (Serial: %s)', telemetry['serial'] if 'serial' in telemetry else 'N/A')
-            self.loggerObj.info('Telemetry received (Serial: %s)', telemetry['serial'] if 'serial' in telemetry else 'N/A')
-            # Check whether the telemetry is plausible
-            telemetry = self.telemetryChecks.check_plausibility(self, telemetry)
-            self.loggerObj.debug('Plausibility checks performed (Serial: %s)', telemetry['serial'] if 'serial' in telemetry else 'N/A')
-            # Optionally write the telemetry
-            if self.writet:
-                if 'serial' in telemetry:
-                    self.writeData.write_telemetry(self, telemetry)
+            # Perform CRC
+            if self.crc_calculator.verify(aprs_package[:-2], aprs_package[-1] << 8 | aprs_package[-2]):
+                # Parse the APRS package
+                telemetry = self.handleData.parse_aprs_package(self, aprs_package[:-2])
+                self.loggerObj.debug('APRS package parsed (Serial: %s)', telemetry['serial'] if 'serial' in telemetry else 'N/A')
+                self.loggerObj.info('Telemetry received (Serial: %s)', telemetry['serial'] if 'serial' in telemetry else 'N/A')
+                # Check whether the telemetry is plausible
+                telemetry = self.telemetryChecks.check_plausibility(self, telemetry)
+                self.loggerObj.debug('Plausibility checks performed (Serial: %s)', telemetry['serial'] if 'serial' in telemetry else 'N/A')
+                # Optionally write the telemetry
+                if self.writet:
+                    if 'serial' in telemetry:
+                        self.writeData.write_telemetry(self, telemetry)
+                    else:
+                        self.loggerObj.error('Could not write telemetry (serial missing)')
+                # Check whether the mandatory telemetry for SondeHub is included
+                if self.telemetryChecks.check_mandatory(self, telemetry):
+                    self.loggerObj.debug('Mandatory data check successful (Serial: %s)', telemetry['serial'])
+                    # Reformat the telemetry to the SondeHub telemetry format
+                    reformatted_telemetry = self.handleData.reformat_telemetry(self, telemetry)
+                    self.loggerObj.debug('Telemetry reformatted (Serial: %s)', reformatted_telemetry['serial'])
+                    # Optionally write the reformatted telemetry
+                    if self.writer:
+                        self.writeData.write_reformatted_telemetry(self, reformatted_telemetry)
+                    # Store the reformatted telemetry to the upload queue
+                    try:
+                        self.upload_queue.put(reformatted_telemetry, False)
+                        self.loggerObj.debug('Reformatted telemetry put in queue (Serial: %s)', reformatted_telemetry['serial'])
+                    except queue.Full:
+                        self.loggerObj.warning('Upload queue full')
                 else:
-                    self.loggerObj.error('Could not write telemetry (serial missing)')
-            # Check whether the mandatory telemetry for SondeHub is included
-            if self.telemetryChecks.check_mandatory(self, telemetry):
-                self.loggerObj.debug('Mandatory data check successful (Serial: %s)', telemetry['serial'])
-                # Reformat the telemetry to the SondeHub telemetry format
-                reformatted_telemetry = self.handleData.reformat_telemetry(self, telemetry)
-                self.loggerObj.debug('Telemetry reformatted (Serial: %s)', reformatted_telemetry['serial'])
-                # Optionally write the reformatted telemetry
-                if self.writer:
-                    self.writeData.write_reformatted_telemetry(self, reformatted_telemetry)
-                # Store the reformatted telemetry to the upload queue
-                try:
-                    self.upload_queue.put(reformatted_telemetry, False)
-                    self.loggerObj.debug('Reformatted telemetry put in queue (Serial: %s)', reformatted_telemetry['serial'])
-                except queue.Full:
-                    self.loggerObj.warning('Upload queue full')
+                    self.loggerObj.error('Mandatory data check failed (Serial: %s)', telemetry['serial'] if 'serial' in telemetry else 'N/A')
             else:
-                self.loggerObj.error('Mandatory data check failed (Serial: %s)', telemetry['serial'] if 'serial' in telemetry else 'N/A')
+                self.loggerObj.error('APRS package CRC failed')
 
 
 # Upload the telemetry packages

SondeHubUploader/uploader.py

@@ -9,7 +9,7 @@
 import requests
 import json
 import gzip
-from email.utils import formatdate
+import email.utils
 
 
 # Upload station to SondeHub
@@ -36,7 +36,7 @@ def upload_station(self):
             headers = {
                 'User-Agent': self.shuConfig.software_name + '-' + self.shuConfig.software_version,
                 'Content-Type': 'application/json',
-                'Date': formatdate(timeval=None, localtime=False, usegmt=True)
+                'Date': email.utils.formatdate(timeval=None, localtime=False, usegmt=True)
             }
             req = requests.put(
                 self.shuConfig.sondehub_station_url,
@@ -91,7 +91,7 @@ def upload_telemetry(self, reformatted_telemetry):
                 'User-Agent': self.shuConfig.software_name + '-' + self.shuConfig.software_version,
                 'Content-Encoding': 'gzip',
                 'Content-Type': 'application/json',
-                'Date': formatdate(timeval=None, localtime=False, usegmt=True)
+                'Date': email.utils.formatdate(timeval=None, localtime=False, usegmt=True)
             }
             req = requests.put(
                 self.shuConfig.sondehub_telemetry_url,

SondeHubUploader/utils.py

@@ -7,7 +7,7 @@
 
 import datetime
 import hashlib
-from dateutil.parser import parse
+import dateutil.parser
 
 
 # Search for a key inside an aprs package string
@@ -62,11 +62,11 @@ def fix_datetime(self, hour, minute, second, leap, local_datetime_str=None):
         now = datetime.datetime.utcnow()
         self.loggerObj.debug('Using UTC for datetime fixing (%s)', now)
     else:
-        now = parse(local_datetime_str)
+        now = dateutil.parser.parse(local_datetime_str)
         self.loggerObj.debug('Using local datetime string for datetime fixing (%s)', now)
 
     # A datetime string is generated, using the radiosonde time and the current date
-    fixed_datetime = parse(f'{hour:02d}:{minute:02d}:{second:02d}', default=now)
+    fixed_datetime = dateutil.parser.parse(f'{hour:02d}:{minute:02d}:{second:02d}', default=now)
 
     # Everything is fine, if the time is outside the rollover window
     if now.hour in [23, 0]:

SondeHubUploader/writeData.py

@@ -7,6 +7,7 @@
 
 # Modules
 import csv
+import datetime
 import os.path
 
 
@@ -17,7 +18,7 @@ def write_raw_data(self, raw_data):
     filename = self.filepath + '/' + self.shuConfig.filename_raw_data + '.txt'
     try:
         f = open(filename, 'a', newline='', encoding='utf-8')
-        f.write(str(raw_data))
+        f.write('[' + str(datetime.datetime.now()) + '] ' + str(raw_data))
         # All entries are separated by a new line
         f.write('\n')
         f.close()

dxlAPRS-SHUE.py

@@ -9,8 +9,8 @@
 import copy
 import time
 
-import parameterChecks
 # Own modules
+import parameterChecks
 import mainConfig
 import procName
 import parameterHandling
@@ -44,7 +44,7 @@
         if any(not element for element in checked_parameters.values()):
             # Print an empty line for better readability
             print()
-        
+
         print('Running...')
 
         # Create a 'SondeHubUploader' object