From 52f793dae60e65faab2ee3013d4fcb32c8bd830d Mon Sep 17 00:00:00 2001 From: Joshua Reed <11220408+jreed1701@users.noreply.github.com> Date: Thu, 28 Mar 2024 17:57:00 -0600 Subject: [PATCH 1/4] Add rotating log to file. --- application/api/v1/blueprints/architect.py | 20 ++++++--- application/common/constants.py | 10 +++++ application/factory.py | 51 +++++++++++++++++++++- requirements.txt | 1 + 4 files changed, 74 insertions(+), 8 deletions(-) diff --git a/application/api/v1/blueprints/architect.py b/application/api/v1/blueprints/architect.py index 6747bda..5df4cdb 100644 --- a/application/api/v1/blueprints/architect.py +++ b/application/api/v1/blueprints/architect.py @@ -37,12 +37,20 @@ def agent_info(): # TODO - Tech Debt: Update agent info page to get this info over websocket. Works for now # but does not scale. for game in games: - update_dict = steam_mgr.is_update_required( - game["game_steam_build_id"], - game["game_steam_build_branch"], - game["game_steam_id"], - ) - game["update_required"] = update_dict["is_required"] + game_steam_id = game["game_steam_id"] + try: + update_dict = steam_mgr.is_update_required( + game["game_steam_build_id"], + game["game_steam_build_branch"], + game_steam_id, + ) + game["update_required"] = update_dict["is_required"] + except Exception: + game["update_required"] = "ERROR" + logger.error( + f"Unable to retrieve game info for game_steam_id: {game_steam_id}", + exc_info=True + ) info: dict = platform_dict info.update({"games": games}) diff --git a/application/common/constants.py b/application/common/constants.py index c265bef..d4ddd32 100644 --- a/application/common/constants.py +++ b/application/common/constants.py @@ -1,3 +1,4 @@ +import logging import platform from datetime import datetime @@ -67,6 +68,15 @@ class FileModes(Enum): WAIT_FOR_BACKEND: int = 1 FLASK_SERVER_PORT: int = 5000 MILIS_PER_SECOND = 1000 +BYTES_PER_KB = 1024 +KB_PER_MB = 1024 + +# Logging +DEFAULT_LOG_LEVEL = logging.NOTSET +DEFAULT_LOG_DATE_FORMAT = "%m/%d/%Y %I:%M:%S %p" +DEFAULT_LOG_FORMAT = '%(filename)s:%(lineno)s %(levelname)s:%(message)s' +#DEFAULT_LOG_SIZE_BYTES = 1024 * KB_PER_MB * 64 # MB +DEFAULT_LOG_SIZE_BYTES = 1024 # Controls not meant to be hooked up to GUI. Just for dev/debug: ENABLE_TIMEIT_PRINTS = False diff --git a/application/factory.py b/application/factory.py index ff4f4f9..3b86084 100644 --- a/application/factory.py +++ b/application/factory.py @@ -1,7 +1,11 @@ +import logging import os +import shutil +import sys from alembic import command from alembic.config import Config +from concurrent_log_handler import ConcurrentRotatingFileHandler from flask import Flask from application.common import logger, constants, toolbox @@ -64,13 +68,54 @@ def _handle_migrations(flask_app: Flask): command.upgrade(alembic_cfg, "head") +def _handle_logging(): + """Update log configuration.""" + # Remove all handlers associated with the root logger object. + for handler in logging.root.handlers[:]: + logging.root.removeHandler(handler) + + log_file_folder = os.path.join( + constants.DEFAULT_INSTALL_PATH, + "logs", + ) + + # Remove old log directory. + if os.path.exists(log_file_folder): + shutil.rmtree(log_file_folder) + + # Create the directory + os.makedirs(log_file_folder, exist_ok=True) + + log_file_full_path = os.path.join(log_file_folder, "agent-smith-log.txt") + + # Reconfigure logging again, this time with a file in addition to stdout. + formatter = logging.Formatter(constants.DEFAULT_LOG_FORMAT) + logger_level = constants.DEFAULT_LOG_LEVEL + + # File handler + file_handler = ConcurrentRotatingFileHandler( + log_file_full_path, + mode='a', + encoding='utf-8', + maxBytes=constants.DEFAULT_LOG_SIZE_BYTES, + backupCount=5 + ) + file_handler.setLevel(logger_level) + file_handler.setFormatter(formatter) + + # Also route to stdout + stdout_handler = logging.StreamHandler(sys.stdout) + stdout_handler.setLevel(logger_level) + stdout_handler.setFormatter(formatter) + + logger.addHandler(stdout_handler) + logger.addHandler(file_handler) def create_app(config=None): - logger.info("Begin initialization.") if config is None: config = DefaultConfig("python") - logger.critical("WARNING. Missing Configuration. Initializing with default...") + logger.info("WARNING. Missing Configuration. Initializing with default...") flask_app = Flask( constants.APP_NAME, @@ -133,6 +178,8 @@ def create_app(config=None): # Run other startup checks. _startup_checks() + _handle_logging() + logger.info(f"{constants.APP_NAME} has been successfully created.") return flask_app diff --git a/requirements.txt b/requirements.txt index 4241188..c2c17c8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ alembic +concurrent-log-handler debugpy flask Flask-SQLAlchemy From 80e9024a54fb5779f441496f429e28fe91bfd5c1 Mon Sep 17 00:00:00 2001 From: Joshua Reed <11220408+jreed1701@users.noreply.github.com> Date: Thu, 28 Mar 2024 17:57:35 -0600 Subject: [PATCH 2/4] Update log size limit --- application/common/constants.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/application/common/constants.py b/application/common/constants.py index d4ddd32..39d7f23 100644 --- a/application/common/constants.py +++ b/application/common/constants.py @@ -75,8 +75,7 @@ class FileModes(Enum): DEFAULT_LOG_LEVEL = logging.NOTSET DEFAULT_LOG_DATE_FORMAT = "%m/%d/%Y %I:%M:%S %p" DEFAULT_LOG_FORMAT = '%(filename)s:%(lineno)s %(levelname)s:%(message)s' -#DEFAULT_LOG_SIZE_BYTES = 1024 * KB_PER_MB * 64 # MB -DEFAULT_LOG_SIZE_BYTES = 1024 +DEFAULT_LOG_SIZE_BYTES = 1024 * KB_PER_MB * 64 # MB # Controls not meant to be hooked up to GUI. Just for dev/debug: ENABLE_TIMEIT_PRINTS = False From 0d008f1c86a26d5aca9707be0f6fa36bb4ea449d Mon Sep 17 00:00:00 2001 From: Joshua Reed <11220408+jreed1701@users.noreply.github.com> Date: Thu, 28 Mar 2024 18:00:38 -0600 Subject: [PATCH 3/4] Update formatting. --- application/api/v1/blueprints/architect.py | 2 +- application/common/constants.py | 4 ++-- application/factory.py | 9 +++++---- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/application/api/v1/blueprints/architect.py b/application/api/v1/blueprints/architect.py index 5df4cdb..2da8231 100644 --- a/application/api/v1/blueprints/architect.py +++ b/application/api/v1/blueprints/architect.py @@ -49,7 +49,7 @@ def agent_info(): game["update_required"] = "ERROR" logger.error( f"Unable to retrieve game info for game_steam_id: {game_steam_id}", - exc_info=True + exc_info=True, ) info: dict = platform_dict diff --git a/application/common/constants.py b/application/common/constants.py index 39d7f23..b99ba39 100644 --- a/application/common/constants.py +++ b/application/common/constants.py @@ -74,8 +74,8 @@ class FileModes(Enum): # Logging DEFAULT_LOG_LEVEL = logging.NOTSET DEFAULT_LOG_DATE_FORMAT = "%m/%d/%Y %I:%M:%S %p" -DEFAULT_LOG_FORMAT = '%(filename)s:%(lineno)s %(levelname)s:%(message)s' -DEFAULT_LOG_SIZE_BYTES = 1024 * KB_PER_MB * 64 # MB +DEFAULT_LOG_FORMAT = "%(filename)s:%(lineno)s %(levelname)s:%(message)s" +DEFAULT_LOG_SIZE_BYTES = 1024 * KB_PER_MB * 64 # MB # Controls not meant to be hooked up to GUI. Just for dev/debug: ENABLE_TIMEIT_PRINTS = False diff --git a/application/factory.py b/application/factory.py index 3b86084..4e2e6ba 100644 --- a/application/factory.py +++ b/application/factory.py @@ -68,6 +68,7 @@ def _handle_migrations(flask_app: Flask): command.upgrade(alembic_cfg, "head") + def _handle_logging(): """Update log configuration.""" # Remove all handlers associated with the root logger object. @@ -95,10 +96,10 @@ def _handle_logging(): # File handler file_handler = ConcurrentRotatingFileHandler( log_file_full_path, - mode='a', - encoding='utf-8', + mode="a", + encoding="utf-8", maxBytes=constants.DEFAULT_LOG_SIZE_BYTES, - backupCount=5 + backupCount=5, ) file_handler.setLevel(logger_level) file_handler.setFormatter(formatter) @@ -111,8 +112,8 @@ def _handle_logging(): logger.addHandler(stdout_handler) logger.addHandler(file_handler) -def create_app(config=None): +def create_app(config=None): if config is None: config = DefaultConfig("python") logger.info("WARNING. Missing Configuration. Initializing with default...") From f54a0a61764794b6b23ffc24b6fa797b3625cdf0 Mon Sep 17 00:00:00 2001 From: Joshua Reed <11220408+jreed1701@users.noreply.github.com> Date: Fri, 29 Mar 2024 10:59:29 -0600 Subject: [PATCH 4/4] Make log level configurable and update dev and prod defaults. --- application/config/config.py | 2 ++ application/factory.py | 9 ++++----- application/gui/launch.py | 7 +++++-- server.py | 4 ++-- 4 files changed, 13 insertions(+), 9 deletions(-) diff --git a/application/config/config.py b/application/config/config.py index ee9be5d..579b3c1 100644 --- a/application/config/config.py +++ b/application/config/config.py @@ -1,3 +1,4 @@ +import logging import os import platform import uuid @@ -19,6 +20,7 @@ class DefaultConfig: FLASK_RUN_PORT = "5000" FLASK_FORCE_AUTH = False # Leave as False except in testing. FLASK_DISABLE_AUTH = False + LOG_LEVEL = logging.NOTSET # NGINX Settings NGINX_DEFAULT_HOSTNAME = "localhost" diff --git a/application/factory.py b/application/factory.py index 4e2e6ba..8708449 100644 --- a/application/factory.py +++ b/application/factory.py @@ -69,7 +69,7 @@ def _handle_migrations(flask_app: Flask): command.upgrade(alembic_cfg, "head") -def _handle_logging(): +def _handle_logging(logger_level=constants.DEFAULT_LOG_LEVEL): """Update log configuration.""" # Remove all handlers associated with the root logger object. for handler in logging.root.handlers[:]: @@ -91,7 +91,6 @@ def _handle_logging(): # Reconfigure logging again, this time with a file in addition to stdout. formatter = logging.Formatter(constants.DEFAULT_LOG_FORMAT) - logger_level = constants.DEFAULT_LOG_LEVEL # File handler file_handler = ConcurrentRotatingFileHandler( @@ -99,7 +98,7 @@ def _handle_logging(): mode="a", encoding="utf-8", maxBytes=constants.DEFAULT_LOG_SIZE_BYTES, - backupCount=5, + backupCount=10, ) file_handler.setLevel(logger_level) file_handler.setFormatter(formatter) @@ -116,7 +115,7 @@ def _handle_logging(): def create_app(config=None): if config is None: config = DefaultConfig("python") - logger.info("WARNING. Missing Configuration. Initializing with default...") + logger.warning("WARNING. Missing Configuration. Initializing with default...") flask_app = Flask( constants.APP_NAME, @@ -179,7 +178,7 @@ def create_app(config=None): # Run other startup checks. _startup_checks() - _handle_logging() + _handle_logging(logger_level=config.LOG_LEVEL) logger.info(f"{constants.APP_NAME} has been successfully created.") diff --git a/application/gui/launch.py b/application/gui/launch.py index c925d74..35c3564 100644 --- a/application/gui/launch.py +++ b/application/gui/launch.py @@ -1,3 +1,4 @@ +import logging import os import time @@ -54,9 +55,11 @@ def _create_backend(self) -> Flask: config = DefaultConfig("python") config.obtain_environment_variables() + config.DEBUG = False + config.LOG_LEVEL = logging.INFO + config.ENV = "production" + app = create_app(config=config) - app.debug = False - app.config["ENV"] = "production" return app diff --git a/server.py b/server.py index 89d21ee..86b680e 100755 --- a/server.py +++ b/server.py @@ -1,7 +1,6 @@ #!/usr/bin/env python -# -*- coding: utf-8 -*- - import argparse as _argparse +import logging import sys as _sys from application.config.config import DefaultConfig @@ -45,6 +44,7 @@ def apply(self): config.DEBUG = True config.ENV = "development" + config.LOG_LEVEL = logging.DEBUG self.app = create_app(config=config)