Feature/telemetry (#67)

* Commit subject: Update models and token limits in LanguageModel class Commit description: * Refactor main.py and monkey.py Description: * Commit subject: Ignore monkey_patch, trackers, models, and src cache files in .gitignore Commit description: * Remove pycache files Description: * json dumps for data to endpoint * Remove outdated cache files and deleted unnecessary files in the feature/telemetry branch. Remove outdated cache files and deleted unnecessary files --------- Co-authored-by: Jack Hopkins <[email protected]> Co-authored-by: Ben Magolan <[email protected]>
Tanuki · Nov 18, 2023 · b011f8d · b011f8d
1 parent 03c5fa7
commit b011f8d
Show file tree

Hide file tree

Showing 4 changed files with 34 additions and 7 deletions.
diff --git a/.gitignore b/.gitignore
@@ -6,6 +6,7 @@ __pycache__/
 .env
 /.env
 /tests/test_load/bloom_filter_state.bin
+src/monkey_patch/__pycache__/*
 src/trackers/__pycache__/*
 src/models/__pycache__/*
 src/__pycache__/*

diff --git a/examples/todolist/backend/main.py b/examples/todolist/backend/main.py
@@ -27,10 +27,13 @@ async def define_behavior():
     """
 
     assert create_todolist_item("I would like to go to the store and buy some milk") \
-           == TodoItem(goal="Go to the store and buy some milk", people=["Me"])
+           == TodoItem(goal="Go to the store and buy some milk",
+                       people=["Me"])
 
     assert create_todolist_item("I need to go and visit Jeff at 3pm tomorrow") \
-           == TodoItem(goal="Go and visit Jeff", people=["Me"], deadline=datetime.datetime(2021, 1, 1, 15, 0))
+           == TodoItem(goal="Go and visit Jeff",
+                       people=["Me"],
+                       deadline=datetime.datetime(2021, 1, 1, 15, 0))
 
 
 @asynccontextmanager

diff --git a/src/monkey_patch/language_models/language_modeler.py b/src/monkey_patch/language_models/language_modeler.py
@@ -13,9 +13,16 @@ def __init__(self, generation_token_limit = 512) -> None:
         self.api_models = {"openai": Openai_API()}
         self.repair_instruction = "Below are an outputs of a function applied to inputs, which failed type validation. The input to the function is brought out in the INPUT section and function description is brought out in the FUNCTION DESCRIPTION section. Your task is to apply the function to the input and return a correct output in the right type. The FAILED EXAMPLES section will show previous outputs of this function applied to the data, which failed type validation and hence are wrong outputs. Using the input and function description output the accurate output following the output_class_definition and output_type_hint attributes of the function description, which define the output type. Make sure the output is an accurate function output and in the correct type. Return None if you can't apply the function to the input or if the output is optional and the correct output is None."
         self.generation_length = generation_token_limit
-        self.models = {"gpt-4":{"token_limit": 8192 - self.generation_length, "type": "openai"},
-                        "gpt-4-32k": {"token_limit": 32768 - self.generation_length, "type": "openai"}
-                        } # models and token counts
+        self.models = {
+            "gpt-4-1106-preview": {
+                "token_limit": 128000 - self.generation_length, "type": "openai"
+            },
+            "gpt-4": {
+                "token_limit": 8192 - self.generation_length, "type": "openai"
+            },
+            "gpt-4-32k": {
+                "token_limit": 32768 - self.generation_length, "type": "openai"}
+            } # models and token counts
 
 
     def generate(self, args, kwargs, function_modeler, function_description, llm_parameters = {}):

diff --git a/src/monkey_patch/monkey.py b/src/monkey_patch/monkey.py
@@ -9,6 +9,8 @@
 from typing import Optional
 from unittest.mock import patch
 
+import requests
+
 from monkey_patch.assertion_visitor import AssertionVisitor
 from monkey_patch.function_modeler import FunctionModeler
 from monkey_patch.language_models.language_modeler import LanguageModel
@@ -77,6 +79,14 @@ class Monkey:
     def _load_alignments(func_hash: str):
         Monkey.function_modeler.load_align_statements(func_hash)
 
+    @staticmethod
+    def _anonymous_usage(*args, **kwargs):
+        """
+        Post anonymously to the usage server so we know what configs are commonly used in the project.
+        :return:
+        """
+        requests.post('https://idhhnusnhkkjkpwkm1fr.monkeypatch.ai/telemetry', data=json.dumps(kwargs))
+
     @staticmethod
     def align(test_func):
         """
@@ -92,7 +102,6 @@ def align(test_func):
         @wraps(test_func)
         def wrapper(*args, **kwargs):
             source = textwrap.dedent(inspect.getsource(test_func))
-            #bytecode = compile(test_func.__code__, "", "exec")
             tree = ast.parse(source)
             _locals = locals()
             visitor = AssertionVisitor(_locals, patch_names=Register.function_names_to_patch())
@@ -198,6 +207,7 @@ def _get_args(func_args, kwarg_names, num_args):
 
     @staticmethod
     def patch(test_func):
+        Monkey._anonymous_usage(logger=Monkey.logger.name)
         function_description = Register.load_function_description(test_func)
         Monkey._load_alignments(function_description.__hash__())
 
@@ -221,7 +231,13 @@ def wrapper(*args, **kwargs):
             valid = validator.check_type(choice_parsed, function_description.output_type_hint)
 
             if not valid:
-                choice, choice_parsed, successful_repair = repair_output(args, kwargs, function_description, output.generated_response, validator, Monkey.function_modeler, Monkey.language_modeler)
+                choice, choice_parsed, successful_repair = repair_output(args,
+                                                                         kwargs,
+                                                                         function_description,
+                                                                         output.generated_response,
+                                                                         validator,
+                                                                         Monkey.function_modeler,
+                                                                         Monkey.language_modeler)
 
                 if not successful_repair:
                     raise TypeError(f"Output type was not valid. Expected an object of type {function_description.output_type_hint}, got '{output.generated_response}'")