From 3ca5bc96a6b3ba435252f6e70decdc686027fc96 Mon Sep 17 00:00:00 2001 From: Sven Seeberg Date: Sat, 23 Nov 2024 13:39:11 +0100 Subject: [PATCH] Set up Celery for Integreat Chat Co-authored-by: JoeyStk <72705147+JoeyStk@users.noreply.github.com> --- .circleci/config.yml | 1 + .gitignore | 3 + integreat_cms/api/v3/chat/chat_bot.py | 63 ---------- integreat_cms/api/v3/chat/user_chat.py | 39 ++---- integreat_cms/api/v3/chat/utils/__init__.py | 3 + integreat_cms/api/v3/chat/utils/chat_bot.py | 111 ++++++++++++++++++ .../api/v3/chat/{ => utils}/zammad_api.py | 20 +++- integreat_cms/core/settings.py | 47 +++++++- integreat_cms/integreat_celery/__init__.py | 5 + integreat_cms/integreat_celery/apps.py | 14 +++ integreat_cms/integreat_celery/celery.py | 39 ++++++ .../integreat_celery/migrations/__init__.py | 0 pyproject.toml | 2 + tests/api/test_api_chat.py | 44 +++++-- tools/run.sh | 3 + 15 files changed, 284 insertions(+), 110 deletions(-) delete mode 100644 integreat_cms/api/v3/chat/chat_bot.py create mode 100644 integreat_cms/api/v3/chat/utils/__init__.py create mode 100644 integreat_cms/api/v3/chat/utils/chat_bot.py rename integreat_cms/api/v3/chat/{ => utils}/zammad_api.py (90%) create mode 100644 integreat_cms/integreat_celery/__init__.py create mode 100644 integreat_cms/integreat_celery/apps.py create mode 100644 integreat_cms/integreat_celery/celery.py create mode 100644 integreat_cms/integreat_celery/migrations/__init__.py diff --git a/.circleci/config.yml b/.circleci/config.yml index f2935e841d..c940e212e0 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -19,6 +19,7 @@ jobs: else python3 -m venv .venv source .venv/bin/activate + pip install --upgrade pip pip install -e .[dev-pinned,pinned] fi - save_cache: diff --git a/.gitignore b/.gitignore index a61ca98cfe..8dc9526001 100644 --- a/.gitignore +++ b/.gitignore @@ -68,3 +68,6 @@ integreat_cms/xliff/download # Postgres folder .postgres + +# Celery +celerybeat-schedule.db diff --git a/integreat_cms/api/v3/chat/chat_bot.py b/integreat_cms/api/v3/chat/chat_bot.py deleted file mode 100644 index da5023279f..0000000000 --- a/integreat_cms/api/v3/chat/chat_bot.py +++ /dev/null @@ -1,63 +0,0 @@ -""" -Wrapper for the Chat Bot / LLM API -""" - -import requests -from django.conf import settings - -from ....cms.models import Region -from ....cms.utils.content_translation_utils import get_public_translation_for_link - - -class ChatBot: - """ - API Wrapper for the LLM / Chat Bot - """ - - def __init__(self, hostname: str = "igchat-inference.tuerantuer.org"): - self.hostname = hostname - - def automatic_answer(self, message: str, region: Region, language_slug: str) -> str: - """ - Get automatic answer to question - """ - url = f"https://{self.hostname}/chatanswers/extract_answer/" - body = {"message": message, "language": language_slug, "region": region.slug} - r = requests.post(url, json=body, timeout=30) - return self.format_message(r.json()) - - def format_message(self, response: dict) -> str: - """ - Transform JSON into readable message - """ - if "answer" not in response or not response["answer"]: - return "" - if "sources" not in response or not response["sources"]: - return response["answer"] - sources = "".join( - [ - f"
  • {title}
    • " - for path in response["sources"] - if ( - title := get_public_translation_for_link(settings.WEBAPP_URL + path) - ) - ] - ) - return f"{response['answer']}\n" - - def automatic_translation( - self, message: str, source_lang_slug: str, target_lang_slug: str - ) -> str: - """ - Use LLM to translate message - """ - url = f"https://{self.hostname}/chatanswers/translate_message/" - body = { - "message": message, - "source_language": source_lang_slug, - "target_language": target_lang_slug, - } - response = requests.post(url, json=body, timeout=30).json() - if "status" in response and response["status"] == "success": - return response["translation"] - return "" diff --git a/integreat_cms/api/v3/chat/user_chat.py b/integreat_cms/api/v3/chat/user_chat.py index bd140de835..31c0f27b83 100644 --- a/integreat_cms/api/v3/chat/user_chat.py +++ b/integreat_cms/api/v3/chat/user_chat.py @@ -16,8 +16,8 @@ from ....cms.models import ABTester, AttachmentMap, Language, Region, UserChat from ...decorators import json_response -from .chat_bot import ChatBot -from .zammad_api import ZammadChatAPI +from .utils.chat_bot import process_answer, process_user_message +from .utils.zammad_api import ZammadChatAPI if TYPE_CHECKING: from django.http import HttpRequest @@ -231,11 +231,8 @@ def zammad_webhook(request: HttpRequest) -> JsonResponse: ) if not region.integreat_chat_enabled: return JsonResponse({"status": "Integreat Chat disabled"}) - client = ZammadChatAPI(region) webhook_message = json.loads(request.body) message_text = webhook_message["article"]["body"] - zammad_chat = UserChat.objects.get(zammad_id=webhook_message["ticket"]["id"]) - chat_bot = ChatBot() actions = [] if webhook_message["article"]["internal"]: @@ -249,34 +246,14 @@ def zammad_webhook(request: HttpRequest) -> JsonResponse: webhook_message["article"]["created_by"]["login"] == "tech+integreat-cms@tuerantuer.org" ): - actions.append("question translation") - client.send_message( - zammad_chat.zammad_id, - chat_bot.automatic_translation( - message_text, zammad_chat.language.slug, region.default_language.slug - ), - True, - True, + actions.append("question translation queued") + process_user_message.apply_async( + args=[message_text, region.slug, webhook_message["ticket"]["id"]] ) - if answer := chat_bot.automatic_answer( - message_text, region, zammad_chat.language.slug - ): - actions.append("automatic answer") - client.send_message( - zammad_chat.zammad_id, - answer, - False, - True, - ) else: - actions.append("answer translation") - client.send_message( - zammad_chat.zammad_id, - chat_bot.automatic_translation( - message_text, region.default_language.slug, zammad_chat.language.slug - ), - False, - True, + actions.append("answer translation queued") + process_answer.apply_async( + args=[message_text, region.slug, webhook_message["ticket"]["id"]] ) return JsonResponse( { diff --git a/integreat_cms/api/v3/chat/utils/__init__.py b/integreat_cms/api/v3/chat/utils/__init__.py new file mode 100644 index 0000000000..15126b51d8 --- /dev/null +++ b/integreat_cms/api/v3/chat/utils/__init__.py @@ -0,0 +1,3 @@ +""" +Utils for the Integreat Chat +""" diff --git a/integreat_cms/api/v3/chat/utils/chat_bot.py b/integreat_cms/api/v3/chat/utils/chat_bot.py new file mode 100644 index 0000000000..146bb5a1f4 --- /dev/null +++ b/integreat_cms/api/v3/chat/utils/chat_bot.py @@ -0,0 +1,111 @@ +""" +Wrapper for the Chat Bot / LLM API +""" + +from __future__ import annotations + +import requests +from celery import shared_task +from django.conf import settings + +from integreat_cms.cms.models import Region, UserChat +from integreat_cms.cms.utils.content_translation_utils import ( + get_public_translation_for_link, +) + +from .zammad_api import ZammadChatAPI + + +def format_message(response: dict) -> str: + """ + Transform JSON into readable message + """ + if "answer" not in response or not response["answer"]: + raise ValueError("Could not format message, no answer attribute in response") + if "sources" not in response or not response["sources"]: + return response["answer"] + sources = "".join( + [ + f"
  • {title}
    • " + for path in response["sources"] + if (title := get_public_translation_for_link(settings.WEBAPP_URL + path)) + ] + ) + return f"{response['answer']}\n" + + +def automatic_answer(message: str, region: Region, language_slug: str) -> str | None: + """ + Get automatic answer to question + """ + url = ( + f"https://{settings.INTEGREAT_CHAT_BACK_END_DOMAIN}/chatanswers/extract_answer/" + ) + body = {"message": message, "language": language_slug, "region": region.slug} + r = requests.post(url, json=body, timeout=120) + return format_message(r.json()) + + +def automatic_translation( + message: str, source_language_slug: str, target_language_slug: str +) -> str: + """ + Use LLM to translate message + """ + url = f"https://{settings.INTEGREAT_CHAT_BACK_END_DOMAIN}/chatanswers/translate_message/" + body = { + "message": message, + "source_language": source_language_slug, + "target_language": target_language_slug, + } + response = requests.post(url, json=body, timeout=120).json() + if "status" in response and response["status"] == "success": + return response["translation"] + raise ValueError("Did not receive success response for translation request.") + + +@shared_task +def process_user_message( + message_text: str, region_slug: str, zammad_ticket_id: int +) -> None: + """ + Process the message from an Integreat App user + """ + zammad_chat = UserChat.objects.get(zammad_id=zammad_ticket_id) + region = Region.objects.get(slug=region_slug) + client = ZammadChatAPI(region) + if translation := automatic_translation( + message_text, zammad_chat.language.slug, region.default_language.slug + ): + client.send_message( + zammad_chat.zammad_id, + translation, + True, + True, + ) + if answer := automatic_answer(message_text, region, zammad_chat.language.slug): + client.send_message( + zammad_chat.zammad_id, + answer, + False, + True, + ) + + +@shared_task +def process_answer(message_text: str, region_slug: str, zammad_ticket_id: int) -> None: + """ + Process automatic or counselor answers + """ + zammad_chat = UserChat.objects.get(zammad_id=zammad_ticket_id) + region = Region.objects.get(slug=region_slug) + client = ZammadChatAPI(region) + if translation := automatic_translation( + message_text, region.default_language.slug, zammad_chat.language.slug + ): + client.send_message( + zammad_chat.zammad_id, + translation, + False, + True, + ) diff --git a/integreat_cms/api/v3/chat/zammad_api.py b/integreat_cms/api/v3/chat/utils/zammad_api.py similarity index 90% rename from integreat_cms/api/v3/chat/zammad_api.py rename to integreat_cms/api/v3/chat/utils/zammad_api.py index 808f37dce9..5f49cd3c83 100644 --- a/integreat_cms/api/v3/chat/zammad_api.py +++ b/integreat_cms/api/v3/chat/utils/zammad_api.py @@ -13,7 +13,7 @@ from requests.exceptions import HTTPError from zammad_py import ZammadAPI -from ....cms.models import AttachmentMap, Region, UserChat +from integreat_cms.cms.models import AttachmentMap, Region, UserChat logger = logging.getLogger(__name__) @@ -166,10 +166,20 @@ def get_messages(self, chat: UserChat) -> dict[str, dict | list[dict]]: # pylint: disable=method-hidden def send_message( - self, chat_id: int, message: str, internal: bool = False, auto: bool = False + self, + chat_id: int, + message: str, + internal: bool = False, + automatic_message: bool = False, ) -> dict: """ Post a new message to the given ticket + + param chat_id: Zammad ID of the chat + param message: The message body + param internal: keep the message internal in Zammad (do not show to user) + param automatic_message: sets title to "automatically generated message" + return: dict with Zammad article data """ params = { "ticket_id": chat_id, @@ -178,9 +188,11 @@ def send_message( "content_type": "text/html", "internal": internal, "subject": ( - "automatically generated message" if auto else "app user message" + "automatically generated message" + if automatic_message + else "app user message" ), - "sender": "Customer" if not auto else "Agent", + "sender": "Customer" if not automatic_message else "Agent", } return self._parse_response( # type: ignore[return-value] self._attempt_call(self.client.ticket_article.create, params=params) diff --git a/integreat_cms/core/settings.py b/integreat_cms/core/settings.py index 634a20c68c..dabcb0a3b3 100644 --- a/integreat_cms/core/settings.py +++ b/integreat_cms/core/settings.py @@ -20,7 +20,8 @@ from django.core.exceptions import ImproperlyConfigured from django.utils.translation import gettext_lazy as _ -from ..nominatim_api.utils import BoundingBox +from integreat_cms.nominatim_api.utils import BoundingBox + from .logging_formatter import ColorFormatter, RequestFormatter from .utils.strtobool import strtobool @@ -372,6 +373,7 @@ "integreat_cms.nominatim_api", "integreat_cms.summ_ai_api", "integreat_cms.textlab_api", + "integreat_cms.integreat_celery", # Installed Django apps "django.contrib.auth", "django.contrib.contenttypes", @@ -1335,3 +1337,46 @@ #: Zammad ticket group used for Integreat chat messages USER_CHAT_TICKET_GROUP: Final[str] = "integreat-chat" + +#: Integreat Chat (app) backend server domain +INTEGREAT_CHAT_BACK_END_DOMAIN = "igchat-inference.tuerantuer.org" + +########## +# CELERY # +########## + +#: Configure Celery to use a custom time zone. The timezone value can be any time zone supported by the pytz library. +#: If not set the UTC timezone is used. For backwards compatibility there is also a CELERY_ENABLE_UTC setting, +#: and this is set to false the system local timezone is used instead. +CELERY_TIMEZONE = "UTC" + +#: If True the task will report its status as ``started`` when the task is executed by a worker. +#: The default value is False as the normal behavior is to not report that level of granularity. +#: Tasks are either pending, finished, or waiting to be retried. +#: Having a ``started`` state can be useful for when there are long running tasks +#: and there’s a need to report what task is currently running. +CELERY_TASK_TRACK_STARTED = True + +#: Task hard time limit in seconds. The worker processing the task will be killed +#: and replaced with a new one when this is exceeded. +CELERY_TASK_TIME_LIMIT = 60 * 60 * 1 + +#: Default broker URL. +CELERY_BROKER_URL = os.environ.get( + "CELERY_REDIS_URL", + ( + "redis+socket:///var/run/redis/redis-server.sock" + if not DEBUG + else "redis://localhost:6379/0" + ), +) + +#: The backend used to store task results (tombstones). Disabled by default. +CELERY_RESULT_BACKEND = os.environ.get( + "CELERY_REDIS_URL", + ( + "redis+socket:///var/run/redis/redis-server.sock" + if not DEBUG + else "redis://localhost:6379/0" + ), +) diff --git a/integreat_cms/integreat_celery/__init__.py b/integreat_cms/integreat_celery/__init__.py new file mode 100644 index 0000000000..5568b6d791 --- /dev/null +++ b/integreat_cms/integreat_celery/__init__.py @@ -0,0 +1,5 @@ +# This will make sure the app is always imported when +# Django starts so that shared_task will use this app. +from .celery import app as celery_app + +__all__ = ("celery_app",) diff --git a/integreat_cms/integreat_celery/apps.py b/integreat_cms/integreat_celery/apps.py new file mode 100644 index 0000000000..d51b317c63 --- /dev/null +++ b/integreat_cms/integreat_celery/apps.py @@ -0,0 +1,14 @@ +""" +Set up celery app +""" + +from django.apps import AppConfig + + +class IntegreatCeleryConfig(AppConfig): + """ + Configuration for Celery + """ + + default_auto_field = "django.db.models.BigAutoField" + name = "integreat_cms.integreat_celery" diff --git a/integreat_cms/integreat_celery/celery.py b/integreat_cms/integreat_celery/celery.py new file mode 100644 index 0000000000..353837c17e --- /dev/null +++ b/integreat_cms/integreat_celery/celery.py @@ -0,0 +1,39 @@ +""" +Celery worker +""" + +import configparser +import os + +from celery import Celery + +# Read config from config file +config = configparser.ConfigParser(interpolation=None) +config.read("/etc/integreat-cms.ini") +for section in config.sections(): + for KEY, VALUE in config.items(section): + os.environ.setdefault(f"INTEGREAT_CMS_{KEY.upper()}", VALUE) +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "integreat_cms.core.settings") +app = Celery("celery_app") +app.config_from_object("django.conf:settings", namespace="CELERY") +app.autodiscover_tasks() + + +# @app.task +# def wrapper_create_statistics(): +# """ +# Periodic task to generate region statistics +# """ +# print("create statistics") +# +# +# @app.on_after_configure.connect +# def setup_periodic_tasks(sender, **kwargs): +# """ +# Set up a periodic job to look for new videos +# """ +# sender.add_periodic_task( +# 84600, +# wrapper_create_statistics.s(), +# name="wrapper_create_statistics", +# ) diff --git a/integreat_cms/integreat_celery/migrations/__init__.py b/integreat_cms/integreat_celery/migrations/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/pyproject.toml b/pyproject.toml index a6bff8cc45..e82c411f6b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,10 +39,12 @@ dependencies = [ "aiohttp", "argon2-cffi", "bcrypt", + "celery", "cffi", "deepl", "Django>=4.2,<5.0", "django-cacheops", + "django-celery", "django-cors-headers", "django-db-mutex", "django-debug-toolbar", diff --git a/tests/api/test_api_chat.py b/tests/api/test_api_chat.py index 2fa116e101..8d87202173 100644 --- a/tests/api/test_api_chat.py +++ b/tests/api/test_api_chat.py @@ -64,7 +64,9 @@ def test_api_chat_incorrect_auth_error(load_test_data: None) -> None: :param load_test_data: The fixture providing the test data (see :meth:`~tests.conftest.load_test_data`) """ mock_api = MagicMock() - with patch("integreat_cms.api.v3.chat.zammad_api.ZammadAPI", return_value=mock_api): + with patch( + "integreat_cms.api.v3.chat.utils.zammad_api.ZammadAPI", return_value=mock_api + ): mock_api.user.me.side_effect = HTTPError() client = Client() @@ -88,7 +90,9 @@ def test_api_chat_first_chat(load_test_data: None) -> None: :param load_test_data: The fixture providing the test data (see :meth:`~tests.conftest.load_test_data`) """ mock_api = MagicMock() - with patch("integreat_cms.api.v3.chat.zammad_api.ZammadAPI", return_value=mock_api): + with patch( + "integreat_cms.api.v3.chat.utils.zammad_api.ZammadAPI", return_value=mock_api + ): mock_api.user.all.return_value = [] mock_api.user.me.return_value = {"login": "bot-user"} mock_api.ticket.create.return_value = {"id": 111} @@ -112,7 +116,9 @@ def test_api_chat_force_new_chat(load_test_data: None) -> None: :param load_test_data: The fixture providing the test data (see :meth:`~tests.conftest.load_test_data`) """ mock_api = MagicMock() - with patch("integreat_cms.api.v3.chat.zammad_api.ZammadAPI", return_value=mock_api): + with patch( + "integreat_cms.api.v3.chat.utils.zammad_api.ZammadAPI", return_value=mock_api + ): mock_api.user.all.return_value = [] mock_api.user.me.return_value = {"login": "bot-user"} mock_api.ticket.create.return_value = {"id": 222} @@ -139,7 +145,9 @@ def test_api_chat_send_message(load_test_data: None) -> None: """ mock_api = MagicMock() previous_chat = UserChat.objects.current_chat(default_kwargs["device_id"]).zammad_id - with patch("integreat_cms.api.v3.chat.zammad_api.ZammadAPI", return_value=mock_api): + with patch( + "integreat_cms.api.v3.chat.utils.zammad_api.ZammadAPI", return_value=mock_api + ): mock_api.user.all.return_value = [] mock_api.user.me.return_value = {"login": "bot-user"} mock_api.ticket_article.create.return_value = {} @@ -166,7 +174,9 @@ def test_api_chat_get_messages_success(load_test_data: None) -> None: :param load_test_data: The fixture providing the test data (see :meth:`~tests.conftest.load_test_data`) """ mock_api = MagicMock() - with patch("integreat_cms.api.v3.chat.zammad_api.ZammadAPI", return_value=mock_api): + with patch( + "integreat_cms.api.v3.chat.utils.zammad_api.ZammadAPI", return_value=mock_api + ): mock_api.user.all.return_value = [] mock_api.user.me.return_value = {"login": "bot-user"} mock_api.ticket.articles.return_value = [] @@ -189,7 +199,9 @@ def test_api_chat_get_messages_failure(load_test_data: None) -> None: :param load_test_data: The fixture providing the test data (see :meth:`~tests.conftest.load_test_data`) """ mock_api = MagicMock() - with patch("integreat_cms.api.v3.chat.zammad_api.ZammadAPI", return_value=mock_api): + with patch( + "integreat_cms.api.v3.chat.utils.zammad_api.ZammadAPI", return_value=mock_api + ): mock_api.user.all.return_value = [] mock_api.user.me.return_value = {"login": "bot-user"} @@ -214,7 +226,9 @@ def test_api_chat_create_attachment_success(load_test_data: None) -> None: :param load_test_data: The fixture providing the test data (see :meth:`~tests.conftest.load_test_data`) """ mock_api = MagicMock() - with patch("integreat_cms.api.v3.chat.zammad_api.ZammadAPI", return_value=mock_api): + with patch( + "integreat_cms.api.v3.chat.utils.zammad_api.ZammadAPI", return_value=mock_api + ): mock_api.user.all.return_value = [] mock_api.user.me.return_value = {"login": "bot-user"} mock_api.ticket.articles.return_value = [ @@ -253,7 +267,9 @@ def test_api_chat_get_attachment_success(load_test_data: None) -> None: :param load_test_data: The fixture providing the test data (see :meth:`~tests.conftest.load_test_data`) """ mock_api = MagicMock() - with patch("integreat_cms.api.v3.chat.zammad_api.ZammadAPI", return_value=mock_api): + with patch( + "integreat_cms.api.v3.chat.utils.zammad_api.ZammadAPI", return_value=mock_api + ): mock_api.user.all.return_value = [] mock_api.user.me.return_value = {"login": "bot-user"} mock_api.ticket_article_attachment.download.return_value = b"\00" @@ -279,7 +295,9 @@ def test_api_chat_get_attachment_incorrect_chat_failure(load_test_data: None) -> :param load_test_data: The fixture providing the test data (see :meth:`~tests.conftest.load_test_data`) """ mock_api = MagicMock() - with patch("integreat_cms.api.v3.chat.zammad_api.ZammadAPI", return_value=mock_api): + with patch( + "integreat_cms.api.v3.chat.utils.zammad_api.ZammadAPI", return_value=mock_api + ): mock_api.user.all.return_value = [] mock_api.user.me.return_value = {"login": "bot-user"} @@ -308,7 +326,9 @@ def test_api_chat_get_attachment_missing_attachment_failure( :param load_test_data: The fixture providing the test data (see :meth:`~tests.conftest.load_test_data`) """ mock_api = MagicMock() - with patch("integreat_cms.api.v3.chat.zammad_api.ZammadAPI", return_value=mock_api): + with patch( + "integreat_cms.api.v3.chat.utils.zammad_api.ZammadAPI", return_value=mock_api + ): mock_api.user.all.return_value = [] mock_api.user.me.return_value = {"login": "bot-user"} @@ -331,7 +351,9 @@ def test_api_chat_ratelimiting(load_test_data: None) -> None: :param load_test_data: The fixture providing the test data (see :meth:`~tests.conftest.load_test_data`) """ mock_api = MagicMock() - with patch("integreat_cms.api.v3.chat.zammad_api.ZammadAPI", return_value=mock_api): + with patch( + "integreat_cms.api.v3.chat.utils.zammad_api.ZammadAPI", return_value=mock_api + ): mock_api.user.all.return_value = [] mock_api.user.me.return_value = {"login": "bot-user"} mock_api.ticket.create.return_value = {"id": 333} diff --git a/tools/run.sh b/tools/run.sh index 88a5ab8bc5..0193842443 100755 --- a/tools/run.sh +++ b/tools/run.sh @@ -46,5 +46,8 @@ done # Show success message once dev server is up listen_for_devserver & +# Run Celery worker process +celery -A integreat_cms.integreat_celery worker -l INFO -B --concurrency=1 & + # Start Integreat CMS development webserver deescalate_privileges integreat-cms-cli runserver "localhost:${INTEGREAT_CMS_PORT}"