dev: Add Line Breaks to worker logs (#407)

* mostly there, need to figure out a way to get array list items on single line

* update list logging to concat if multiple items

* fix uts

* lint

helpers/logging_config.py

@@ -1,3 +1,4 @@
+import json
 from copy import deepcopy
 
 from celery._state import get_current_task
@@ -17,6 +18,19 @@ def add_fields(self, log_record, record, message_dict) -> None:
  log_record["task_name"] = "???"
  log_record["task_id"] = "???"
 
+ def format_json_on_new_lines(self, json_str):
+ # Parse the input JSON string
+ data = json.loads(json_str)
+
+ for key, value in data.items():
+ if isinstance(value, list) and len(value) > 10:
+ # If more than 10 elements in a list, concat to single line
+ data[key] = ", ".join(map(str, value))
+
+ # Convert the parsed JSON data back to a formatted JSON string
+ formatted_json = json.dumps(data, indent=4)
+ return formatted_json
+
 
 class CustomLocalJsonFormatter(BaseLogger):
  def jsonify_log_record(self, log_record) -> str:
@@ -25,9 +39,10 @@ def jsonify_log_record(self, log_record) -> str:
  message = log_record.pop("message")
  exc_info = log_record.pop("exc_info", "")
  content = super().jsonify_log_record(log_record)
+ formatted = super().format_json_on_new_lines(content) if content else None
  if exc_info:
- return f"{levelname}: {message} --- {content}\n{exc_info}"
- return f"{levelname}: {message} --- {content}"
+ return f"{levelname}: {message} \n {formatted}\n{exc_info}"
+ return f"{levelname}: {message} \n {formatted}"
 
 
 class CustomDatadogJsonFormatter(BaseLogger):

helpers/tests/unit/test_logging_config.py

@@ -13,7 +13,7 @@ def test_local_formatter(self):
  log_record = {"levelname": "weird_level", "message": "This is a message"}
  cljf = CustomLocalJsonFormatter()
  res = cljf.jsonify_log_record(log_record)
- assert "weird_level: This is a message --- {}" == res
+ assert "weird_level: This is a message \n {}" == res
 
  def test_local_formatter_with_exc_info(self):
  log_record = {
@@ -23,7 +23,7 @@ def test_local_formatter_with_exc_info(self):
  }
  cljf = CustomLocalJsonFormatter()
  res = cljf.jsonify_log_record(log_record)
- assert "weird_level: This is a message --- {}\nLine\nWith\nbreaks" == res
+ assert "weird_level: This is a message \n {}\nLine\nWith\nbreaks" == res
 
  def test_get_logging_config_dict(self, mocker):
  get_current_env = mocker.patch("helpers.logging_config.get_current_env")