From 6f1677f8946a76d40e97b18378d0a1c8e721aa3c Mon Sep 17 00:00:00 2001 From: Mike Rosseel Date: Thu, 26 Sep 2024 20:04:45 +0200 Subject: [PATCH] Move all comet related code to comets.py --- python/PiFinder/calc_utils.py | 134 +--------------------------------- python/PiFinder/catalogs.py | 36 +++++---- python/PiFinder/comets.py | 130 +++++++++++++++++++++++++++++++++ python/PiFinder/main.py | 2 +- python/PiFinder/ui/console.py | 3 +- 5 files changed, 155 insertions(+), 150 deletions(-) create mode 100644 python/PiFinder/comets.py diff --git a/python/PiFinder/calc_utils.py b/python/PiFinder/calc_utils.py index 0e028226..65d211d1 100755 --- a/python/PiFinder/calc_utils.py +++ b/python/PiFinder/calc_utils.py @@ -2,7 +2,7 @@ import pytz import math import numpy as np -from typing import Tuple, Optional, Dict, Any +from typing import Tuple, Optional from skyfield.api import ( wgs84, Loader, @@ -16,26 +16,11 @@ import PiFinder.utils as utils import json import hashlib -from pathlib import Path - -from skyfield.api import load -from skyfield.data import mpc -from skyfield.constants import GM_SUN_Pitjeva_2005_km3_s2 as GM_SUN -from skyfield.elementslib import osculating_elements_of -from skyfield.api import Topos -from PiFinder.utils import Timer -import numpy as np -from concurrent.futures import ProcessPoolExecutor, as_completed -from functools import partial -from multiprocessing import Manager -from multiprocessing import Pool -import time -import requests -import os import logging logger = logging.getLogger("Catalogs.calc_utils") + class FastAltAz: """ Adapted from example at: @@ -457,119 +442,4 @@ def calc_planets(self, dt): return planet_dict -def process_comet(comet_data, dt) -> Dict[str, Any]: - name, row = comet_data - t = sf_utils.ts.from_datetime(dt) - sun = sf_utils.eph['sun'] - comet = sun + mpc.comet_orbit(row, sf_utils.ts, GM_SUN) - - # print(f"Processing comet: {name}, {sf_utils.observer_loc}") - topocentric = (comet - sf_utils.observer_loc).at(t) - heliocentric = (comet - sun).at(t) - - ra, dec, earth_distance = topocentric.radec(sf_utils.ts.J2000) - sun_distance = heliocentric.radec(sf_utils.ts.J2000)[2] - - mag_g = float(row['magnitude_g']) - mag_k = float(row['magnitude_k']) - mag = mag_g + 2.5 * mag_k * math.log10(sun_distance.au) + \ - 5.0 * math.log10(earth_distance.au) - if mag > 15: - return {} - - alt, az = sf_utils.radec_to_altaz(ra._degrees, dec.degrees, dt, atmos=False) - ra_dec = (ra._degrees, dec.degrees) - ra_dec_pretty = (ra_to_hms(ra._degrees), dec_to_dms(dec.degrees)) - alt_az = (alt, az) - - return { - "name": name, - "radec": ra_dec, - "radec_pretty": ra_dec_pretty, - "altaz": alt_az, - "altaz2": (0, 0), - "altaz3": (alt, az), - "mag": mag, - "earth_distance": earth_distance.au, - "sun_distance": sun_distance.au, - "orbital_elements": None, - "row": row - } - - -def comet_data_download(local_filename, url=mpc.COMET_URL): - try: - # Send a HEAD request to get headers without downloading the entire file - response = requests.head(url) - response.raise_for_status() # Raise an exception for bad responses - - # Try to get the Last-Modified header - last_modified = response.headers.get('Last-Modified') - - if last_modified: - remote_date = datetime.strptime(last_modified, '%a, %d %b %Y %H:%M:%S GMT') - logger.debug(f"Remote Last-Modified: {remote_date}") - - # Check if local file exists and its modification time - if os.path.exists(local_filename): - local_date = datetime.fromtimestamp(os.path.getmtime(local_filename)) - logger.debug(f"Local Last-Modified: {local_date}") - - if remote_date <= local_date: - logger.debug("Local file is up to date. No download needed.") - return False - - # Download the file if it's new or doesn't exist locally - logger.debug("Downloading new file...") - response = requests.get(url) - response.raise_for_status() - - with open(local_filename, 'wb') as f: - f.write(response.content) - - # Set the file's modification time to match the server's last-modified time - os.utime(local_filename, (remote_date.timestamp(), remote_date.timestamp())) - - logger.debug("File downloaded successfully.") - return True - else: - logger.debug("Last-Modified header not available. Downloading file...") - response = requests.get(url) - response.raise_for_status() - - with open(local_filename, 'wb') as f: - f.write(response.content) - - logger.debug("File downloaded successfully.") - return True - - except requests.RequestException as e: - logger.error(f"An error occurred: {e}") - return False - - -def calc_comets(dt, comet_names=None) -> dict: - with Timer("calc_comets()"): - comet_dict: Dict[str, Any] = {} - if sf_utils.observer_loc is None or dt is None: - logger.debug(f"calc_comets can't run: observer loc is None: {sf_utils.observer_loc is None}, dt is None: {dt is None}") - return comet_dict - - with open(utils.comet_file, "rb") as f: - comets_df = mpc.load_comets_dataframe(f) - - comets_df = (comets_df.sort_values('reference') - .groupby('designation', as_index=False).last() - .set_index('designation', drop=False)) - - comet_data = list(comets_df.iterrows()) - - for comet in comet_data: - if comet_names is None or comet[0] in comet_names: - result = process_comet(comet, dt) - if result: - comet_dict[result["name"]] = result - return comet_dict - - sf_utils = Skyfield_utils() diff --git a/python/PiFinder/catalogs.py b/python/PiFinder/catalogs.py index 4d04b9dd..cd780eb2 100644 --- a/python/PiFinder/catalogs.py +++ b/python/PiFinder/catalogs.py @@ -8,12 +8,13 @@ from typing import List, Dict, DefaultDict, Optional, Union from collections import defaultdict import PiFinder.calc_utils as calc_utils +from PiFinder.calc_utils import sf_utils from PiFinder.state import SharedStateObj from PiFinder.db.db import Database from PiFinder.db.objects_db import ObjectsDatabase from PiFinder.db.observations_db import ObservationsDatabase from PiFinder.composite_object import CompositeObject, MagnitudeObject -from PiFinder.calc_utils import sf_utils, calc_comets +import PiFinder.comets as comets from PiFinder.utils import Timer, comet_file logger = logging.getLogger("Catalog") @@ -354,7 +355,8 @@ def filter_objects(self) -> List[CompositeObject]: self.filtered_objects = self.catalog_filter.apply(self.get_objects()) logger.info( - f"FILTERED {self.catalog_code} {len(self.filtered_objects)}/{len(self.get_objects())}" + "FILTERED %s %d/%d", self.catalog_code, len( + self.filtered_objects), len(self.get_objects()) ) self.filtered_objects_seq = self._filtered_objects_to_seq() self.last_filtered = time.time() @@ -462,7 +464,7 @@ def add(self, catalog: Catalog, select: bool = False): self.__catalogs.append(catalog) else: logger.warning( - f"Catalog {catalog.catalog_code} already exists, not replaced (in Catalogs.add)" + "Catalog %s already exists, not replaced (in Catalogs.add)", catalog.catalog_code ) def remove(self, catalog_code: str): @@ -471,7 +473,8 @@ def remove(self, catalog_code: str): self.__catalogs.remove(catalog) return - logger.warning("Catalog %s does not exist, cannot remove", catalog_code) + logger.warning( + "Catalog %s does not exist, cannot remove", catalog_code) def get_codes(self, only_selected: bool = True) -> List[str]: return_list = [] @@ -530,7 +533,7 @@ class TimerCatalog(VirtualCatalog): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - self.initialised = False # + self.initialised = False logger.debug("in init of timercatalog") self.timer: Optional[threading.Timer] = None self.is_running: bool = False @@ -642,7 +645,8 @@ def do_timed_task(self): planet = planet_dict[name] obj.ra, obj.dec = planet["radec"] obj.mag = MagnitudeObject([planet["mag"]]) - obj.const = sf_utils.radec_to_constellation(obj.ra, obj.dec) + obj.const = sf_utils.radec_to_constellation( + obj.ra, obj.dec) obj.mag_str = obj.mag.calc_two_mag_representation() @@ -657,17 +661,18 @@ def __init__(self, dt: datetime.datetime, shared_state: SharedStateObj): def _start_background_init(self, dt): def init_task(): - calc_utils.comet_data_download(comet_file) + comets.comet_data_download(comet_file) self.initialised = self.calc_comet_first_time(dt) with self.virtual_id_lock: - new_low = self.assign_virtual_object_ids(self, self.virtual_id_low) + new_low = self.assign_virtual_object_ids( + self, self.virtual_id_low) self.virtual_id_low = new_low threading.Thread(target=init_task, daemon=True).start() def calc_comet_first_time(self, dt): with Timer("CometCatalog.__init__"): - comet_dict = calc_comets(dt) + comet_dict = comets.calc_comets(dt) if not comet_dict: return False for sequence, (name, comet) in enumerate(comet_dict.items()): @@ -683,8 +688,6 @@ def add_comet(self, sequence: int, name: str, comet: Dict[str, Dict[str, float]] constellation = sf_utils.radec_to_constellation(ra, dec) # desc = f"{comet['radec_pretty']}, AltAZ: {comet['altaz']}\nAltAz2: {comet['altaz2']}\nAltAz3: {comet['altaz3']}\n{comet['radec_pretty']}, Earth distance: {comet['earth_distance']} AU\n" desc = f"Distance to\nEarth: {comet['earth_distance']:.2f} AU\nSun: {comet['sun_distance']:.2f} AU" - # if "Olbers" in name: - # print(comet) mag = MagnitudeObject([comet.get("mag", [])]) obj = CompositeObject.from_dict( @@ -713,12 +716,13 @@ def do_timed_task(self): if not self.initialised: return dt = self.shared_state.datetime() - comet_dict = calc_comets(dt, [x.names[0] for x in self._get_objects()]) + comet_dict = comets.calc_comets( + dt, [x.names[0] for x in self._get_objects()]) if not comet_dict: return for obj in self._get_objects(): name = obj.names[0] - logging.debug("Processing", name) + logger.debug("Processing %s", name) comet = comet_dict.get(name, {}) obj.ra, obj.dec = comet["radec"] obj.mag = MagnitudeObject([comet["mag"]]) @@ -751,7 +755,8 @@ def build(self, shared_state) -> Catalogs: # to speed up repeated searches self.catalog_dicts = {} logger.debug("Loaded %i objects from database", len(composite_objects)) - all_catalogs: Catalogs = self._get_catalogs(composite_objects, catalogs_info) + all_catalogs: Catalogs = self._get_catalogs( + composite_objects, catalogs_info) # Initialize planet catalog with whatever date we have for now # This will be re-initialized on activation of Catalog ui module # if we have GPS lock @@ -773,7 +778,8 @@ def check_catalogs_sequences(self, catalogs: Catalogs): for catalog in catalogs.get_catalogs(only_selected=False): result = catalog.check_sequences() if not result: - logger.error("Duplicate sequence catalog %s!", catalog.catalog_code) + logger.error("Duplicate sequence catalog %s!", + catalog.catalog_code) return False return True diff --git a/python/PiFinder/comets.py b/python/PiFinder/comets.py new file mode 100644 index 00000000..0dceab03 --- /dev/null +++ b/python/PiFinder/comets.py @@ -0,0 +1,130 @@ +from typing import Dict, Any +from datetime import datetime +from skyfield.data import mpc +from skyfield.constants import GM_SUN_Pitjeva_2005_km3_s2 as GM_SUN +from skyfield.elementslib import osculating_elements_of +from PiFinder.utils import Timer, comet_file +from PiFinder.calc_utils import sf_utils, ra_to_hms, dec_to_dms +import requests +import os +import logging +import math + +logger = logging.getLogger("Comets") + + +def process_comet(comet_data, dt) -> Dict[str, Any]: + name, row = comet_data + t = sf_utils.ts.from_datetime(dt) + sun = sf_utils.eph['sun'] + comet = sun + mpc.comet_orbit(row, sf_utils.ts, GM_SUN) + + # print(f"Processing comet: {name}, {sf_utils.observer_loc}") + topocentric = (comet - sf_utils.observer_loc).at(t) + heliocentric = (comet - sun).at(t) + + ra, dec, earth_distance = topocentric.radec(sf_utils.ts.J2000) + sun_distance = heliocentric.radec(sf_utils.ts.J2000)[2] + + mag_g = float(row['magnitude_g']) + mag_k = float(row['magnitude_k']) + mag = mag_g + 2.5 * mag_k * math.log10(sun_distance.au) + \ + 5.0 * math.log10(earth_distance.au) + if mag > 15: + return {} + + alt, az = sf_utils.radec_to_altaz(ra._degrees, dec.degrees, dt, atmos=False) + ra_dec = (ra._degrees, dec.degrees) + ra_dec_pretty = (ra_to_hms(ra._degrees), dec_to_dms(dec.degrees)) + alt_az = (alt, az) + + return { + "name": name, + "radec": ra_dec, + "radec_pretty": ra_dec_pretty, + "altaz": alt_az, + "altaz2": (0, 0), + "altaz3": (alt, az), + "mag": mag, + "earth_distance": earth_distance.au, + "sun_distance": sun_distance.au, + "orbital_elements": None, + "row": row + } + + +def comet_data_download(local_filename, url=mpc.COMET_URL): + try: + # Send a HEAD request to get headers without downloading the entire file + response = requests.head(url) + response.raise_for_status() # Raise an exception for bad responses + + # Try to get the Last-Modified header + last_modified = response.headers.get('Last-Modified') + + if last_modified: + remote_date = datetime.strptime(last_modified, '%a, %d %b %Y %H:%M:%S GMT') + logger.debug(f"Remote Last-Modified: {remote_date}") + + # Check if local file exists and its modification time + if os.path.exists(local_filename): + local_date = datetime.fromtimestamp(os.path.getmtime(local_filename)) + logger.debug(f"Local Last-Modified: {local_date}") + + if remote_date <= local_date: + logger.debug("Local file is up to date. No download needed.") + return False + + # Download the file if it's new or doesn't exist locally + logger.debug("Downloading new file...") + response = requests.get(url) + response.raise_for_status() + + with open(local_filename, 'wb') as f: + f.write(response.content) + + # Set the file's modification time to match the server's last-modified time + os.utime(local_filename, (remote_date.timestamp(), remote_date.timestamp())) + + logger.debug("File downloaded successfully.") + return True + else: + logger.debug("Last-Modified header not available. Downloading file...") + response = requests.get(url) + response.raise_for_status() + + with open(local_filename, 'wb') as f: + f.write(response.content) + + logger.debug("File downloaded successfully.") + return True + + except requests.RequestException as e: + logger.error(f"Error downloading comet data: {e}") + return False + + +def calc_comets(dt, comet_names=None) -> dict: + with Timer("calc_comets()"): + comet_dict: Dict[str, Any] = {} + if sf_utils.observer_loc is None or dt is None: + logger.debug(f"calc_comets can't run: observer loc is None: {sf_utils.observer_loc is None}, dt is None: {dt is None}") + return comet_dict + + with open(comet_file, "rb") as f: + comets_df = mpc.load_comets_dataframe(f) + + comets_df = (comets_df.sort_values('reference') + .groupby('designation', as_index=False).last() + .set_index('designation', drop=False)) + + comet_data = list(comets_df.iterrows()) + + for comet in comet_data: + if comet_names is None or comet[0] in comet_names: + result = process_comet(comet, dt) + if result: + comet_dict[result["name"]] = result + return comet_dict + + diff --git a/python/PiFinder/main.py b/python/PiFinder/main.py index 22b0d361..89fd919b 100644 --- a/python/PiFinder/main.py +++ b/python/PiFinder/main.py @@ -239,7 +239,7 @@ def main( shared_state.set_ui_state(ui_state) shared_state.set_arch(arch) # Normal logger.debug("Ui state in main is" + str(shared_state.ui_state())) - console = UIConsole(display_device, None, shared_state, command_queues, cfg) + console = UIConsole(display_device, None, shared_state, command_queues, cfg, Catalogs([])) console.write("Starting....") console.update() diff --git a/python/PiFinder/ui/console.py b/python/PiFinder/ui/console.py index 40fb27af..5f95766b 100644 --- a/python/PiFinder/ui/console.py +++ b/python/PiFinder/ui/console.py @@ -13,14 +13,13 @@ from PIL import Image from PiFinder.ui.base import UIModule from PiFinder.image_util import convert_image_to_mode -from PiFinder.catalogs import Catalogs class UIConsole(UIModule): __title__ = "CONSOLE" def __init__(self, *args, **kwargs): - super().__init__(*args, catalogs=Catalogs([]), **kwargs) + super().__init__(*args, **kwargs) self.dirty = True self.welcome = True