diff --git a/autogpt/commands/file_operations.py b/autogpt/commands/file_operations.py index 9563883e4170..28198ebae277 100644 --- a/autogpt/commands/file_operations.py +++ b/autogpt/commands/file_operations.py @@ -6,6 +6,7 @@ import os.path from typing import Dict, Generator, Literal, Tuple +import charset_normalizer import requests from colorama import Back, Fore from requests.adapters import HTTPAdapter, Retry @@ -153,9 +154,10 @@ def read_file(filename: str) -> str: str: The contents of the file """ try: - with open(filename, "r", encoding="utf-8") as f: - content = f.read() - return content + charset_match = charset_normalizer.from_path(filename).best() + encoding = charset_match.encoding + logger.debug(f"Read file '{filename}' with encoding '{encoding}'") + return str(charset_match) except Exception as err: return f"Error: {err}" diff --git a/requirements.txt b/requirements.txt index 11e951151cd8..e1900ef5360f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -21,6 +21,7 @@ webdriver-manager jsonschema tweepy click +charset-normalizer>=3.1.0 spacy>=3.0.0,<4.0.0 en-core-web-sm @ https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.5.0/en_core_web_sm-3.5.0-py3-none-any.whl