add usage and cost logging

Author: KeremTurgutlu

README.md

@@ -259,7 +259,7 @@ The result of 47 + 23 is 70.
 - model: `claude-sonnet-4-20250514`
 - finish_reason: `stop`
 - usage:
-  `Usage(completion_tokens=17, prompt_tokens=533, total_tokens=550, completion_tokens_details=None, prompt_tokens_details=PromptTokensDetailsWrapper(audio_tokens=None, cached_tokens=0, text_tokens=None, image_tokens=None), cache_creation_input_tokens=0, cache_read_input_tokens=0)`
+  `Usage(completion_tokens=18, prompt_tokens=573, total_tokens=591, completion_tokens_details=None, prompt_tokens_details=PromptTokensDetailsWrapper(audio_tokens=None, cached_tokens=0, text_tokens=None, image_tokens=None), cache_creation_input_tokens=0, cache_read_input_tokens=0)`
 
 </details>
 
@@ -312,15 +312,18 @@ Now I’ll add the result (70) to the third number (59):
      'name': 'add_numbers',
      'content': '129'}
 
-The answer is 129. So 47 + 23 + 59 = 129.
+The answer is **129**.
+
+I calculated this by first adding 47 + 23 = 70, then adding 70 + 59 =
+129.
 
 <details>
 
 - id: `chatcmpl-xxx`
 - model: `claude-sonnet-4-20250514`
 - finish_reason: `stop`
 - usage:
-  `Usage(completion_tokens=25, prompt_tokens=662, total_tokens=687, completion_tokens_details=None, prompt_tokens_details=PromptTokensDetailsWrapper(audio_tokens=None, cached_tokens=0, text_tokens=None, image_tokens=None), cache_creation_input_tokens=0, cache_read_input_tokens=0)`
+  `Usage(completion_tokens=41, prompt_tokens=702, total_tokens=743, completion_tokens_details=None, prompt_tokens_details=PromptTokensDetailsWrapper(audio_tokens=None, cached_tokens=0, text_tokens=None, image_tokens=None), cache_creation_input_tokens=0, cache_read_input_tokens=0)`
 
 </details>
 

cachy.jsonl

@@ -50,4 +50,16 @@
                               'lisette.core.patch_litellm': ('core.html#patch_litellm', 'lisette/core.py'),
                               'lisette.core.random_tool_id': ('core.html#random_tool_id', 'lisette/core.py'),
                               'lisette.core.remove_cache_ckpts': ('core.html#remove_cache_ckpts', 'lisette/core.py'),
-                              'lisette.core.stream_with_complete': ('core.html#stream_with_complete', 'lisette/core.py')}}}
+                              'lisette.core.stream_with_complete': ('core.html#stream_with_complete', 'lisette/core.py')},
+            'lisette.usage': { 'lisette.usage.LisetteUsageLogger': ('usage.html#lisetteusagelogger', 'lisette/usage.py'),
+                               'lisette.usage.LisetteUsageLogger.__init__': ('usage.html#lisetteusagelogger.__init__', 'lisette/usage.py'),
+                               'lisette.usage.LisetteUsageLogger._log_usage': ( 'usage.html#lisetteusagelogger._log_usage',
+                                                                                'lisette/usage.py'),
+                               'lisette.usage.LisetteUsageLogger.async_log_success_event': ( 'usage.html#lisetteusagelogger.async_log_success_event',
+                                                                                             'lisette/usage.py'),
+                               'lisette.usage.LisetteUsageLogger.log_success_event': ( 'usage.html#lisetteusagelogger.log_success_event',
+                                                                                       'lisette/usage.py'),
+                               'lisette.usage.LisetteUsageLogger.user_id_fn': ( 'usage.html#lisetteusagelogger.user_id_fn',
+                                                                                'lisette/usage.py'),
+                               'lisette.usage.Usage': ('usage.html#usage', 'lisette/usage.py'),
+                               'lisette.usage.Usage.total_cost': ('usage.html#usage.total_cost', 'lisette/usage.py')}}}

lisette/_modidx.py

@@ -0,0 +1,38 @@
+"""Lisette usage and cost monitoring"""
+
+# AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/01_usage.ipynb.
+
+# %% auto 0
+__all__ = ['Usage', 'LisetteUsageLogger']
+
+# %% ../nbs/01_usage.ipynb
+from litellm.integrations.custom_logger import CustomLogger
+import time
+try: 
+    from fastlite import *
+    from fastlite.core import dataclass
+except ImportError: raise ImportError("Please install `fastlite` to use sqlite based lisette usage logging.")
+
+# %% ../nbs/01_usage.ipynb
+class Usage: id:int; timestamp:float; model:str; user_id:str; prompt_tokens:int; completion_tokens:int; total_tokens:int; cached_tokens:int; cache_creation_tokens:int; cache_read_tokens:int; web_search_requests:int; response_cost:int
+
+# %% ../nbs/01_usage.ipynb
+class LisetteUsageLogger(CustomLogger):
+    def __init__(self, db_path): 
+        self.db = Database(db_path)
+        self.usage = self.db.create(Usage)
+    
+    async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): self._log_usage(response_obj, kwargs['response_cost'], start_time, end_time)
+    def log_success_event(self, kwargs, response_obj, start_time, end_time):             self._log_usage(response_obj, kwargs['response_cost'], start_time, end_time)
+    def _log_usage(self, response_obj, response_cost, start_time, end_time):
+        usage = response_obj.usage
+        ptd   = usage.prompt_tokens_details
+        self.usage.insert(Usage(timestamp=time.time(), model=response_obj.model, user_id=self.user_id_fn(), prompt_tokens=usage.prompt_tokens, completion_tokens=usage.completion_tokens,
+                                    total_tokens=usage.total_tokens, cached_tokens=ptd.cached_tokens if ptd else 0, cache_creation_tokens=usage.cache_creation_input_tokens, 
+                                    cache_read_tokens=usage.cache_read_input_tokens, web_search_requests=nested_idx(usage, 'server_tool_use', 'web_search_requests'), response_cost=response_cost))
+                  
+    def user_id_fn(self): raise NotImplementedError('Please implement `LisetteUsageLogger.user_id_fn` before initializing, e.g using fastcore.patch.')
+
+# %% ../nbs/01_usage.ipynb
+@patch
+def total_cost(self:Usage, sc=0.01): return self.response_cost + sc * ifnone(self.web_search_requests, 0)

lisette/usage.py

@@ -773,12 +773,12 @@
        "- id: `chatcmpl-xxx`\n",
        "- model: `claude-sonnet-4-5-20250929`\n",
        "- finish_reason: `stop`\n",
-       "- usage: `Usage(completion_tokens=5, prompt_tokens=3, total_tokens=8, completion_tokens_details=None, prompt_tokens_details=PromptTokensDetailsWrapper(audio_tokens=None, cached_tokens=0, text_tokens=None, image_tokens=None), cache_creation_input_tokens=2070, cache_read_input_tokens=0)`\n",
+       "- usage: `Usage(completion_tokens=5, prompt_tokens=2073, total_tokens=2078, completion_tokens_details=None, prompt_tokens_details=PromptTokensDetailsWrapper(audio_tokens=None, cached_tokens=2070, text_tokens=None, image_tokens=None), cache_creation_input_tokens=0, cache_read_input_tokens=2070)`\n",
        "\n",
        "</details>"
       ],
       "text/plain": [
-       "ModelResponse(id='chatcmpl-xxx', created=1000000000, model='claude-sonnet-4-5-20250929', object='chat.completion', system_fingerprint=None, choices=[Choices(finish_reason='stop', index=0, message=Message(content='1', role='assistant', tool_calls=None, function_call=None, provider_specific_fields={'citations': None, 'thinking_blocks': None}))], usage=Usage(completion_tokens=5, prompt_tokens=3, total_tokens=8, completion_tokens_details=None, prompt_tokens_details=PromptTokensDetailsWrapper(audio_tokens=None, cached_tokens=0, text_tokens=None, image_tokens=None), cache_creation_input_tokens=2070, cache_read_input_tokens=0))"
+       "ModelResponse(id='chatcmpl-xxx', created=1000000000, model='claude-sonnet-4-5-20250929', object='chat.completion', system_fingerprint=None, choices=[Choices(finish_reason='stop', index=0, message=Message(content='1', role='assistant', tool_calls=None, function_call=None, provider_specific_fields={'citations': None, 'thinking_blocks': None}))], usage=Usage(completion_tokens=5, prompt_tokens=2073, total_tokens=2078, completion_tokens_details=None, prompt_tokens_details=PromptTokensDetailsWrapper(audio_tokens=None, cached_tokens=2070, text_tokens=None, image_tokens=None), cache_creation_input_tokens=0, cache_read_input_tokens=2070))"
       ]
      },
      "execution_count": null,
@@ -816,12 +816,12 @@
        "- id: `chatcmpl-xxx`\n",
        "- model: `claude-sonnet-4-5-20250929`\n",
        "- finish_reason: `stop`\n",
-       "- usage: `Usage(completion_tokens=5, prompt_tokens=2073, total_tokens=2078, completion_tokens_details=None, prompt_tokens_details=PromptTokensDetailsWrapper(audio_tokens=None, cached_tokens=2070, text_tokens=None, image_tokens=None), cache_creation_input_tokens=2074, cache_read_input_tokens=2070)`\n",
+       "- usage: `Usage(completion_tokens=5, prompt_tokens=4147, total_tokens=4152, completion_tokens_details=None, prompt_tokens_details=PromptTokensDetailsWrapper(audio_tokens=None, cached_tokens=4144, text_tokens=None, image_tokens=None), cache_creation_input_tokens=0, cache_read_input_tokens=4144)`\n",
        "\n",
        "</details>"
       ],
       "text/plain": [
-       "ModelResponse(id='chatcmpl-xxx', created=1000000000, model='claude-sonnet-4-5-20250929', object='chat.completion', system_fingerprint=None, choices=[Choices(finish_reason='stop', index=0, message=Message(content='2', role='assistant', tool_calls=None, function_call=None, provider_specific_fields={'citations': None, 'thinking_blocks': None}))], usage=Usage(completion_tokens=5, prompt_tokens=2073, total_tokens=2078, completion_tokens_details=None, prompt_tokens_details=PromptTokensDetailsWrapper(audio_tokens=None, cached_tokens=2070, text_tokens=None, image_tokens=None), cache_creation_input_tokens=2074, cache_read_input_tokens=2070))"
+       "ModelResponse(id='chatcmpl-xxx', created=1000000000, model='claude-sonnet-4-5-20250929', object='chat.completion', system_fingerprint=None, choices=[Choices(finish_reason='stop', index=0, message=Message(content='2', role='assistant', tool_calls=None, function_call=None, provider_specific_fields={'citations': None, 'thinking_blocks': None}))], usage=Usage(completion_tokens=5, prompt_tokens=4147, total_tokens=4152, completion_tokens_details=None, prompt_tokens_details=PromptTokensDetailsWrapper(audio_tokens=None, cached_tokens=4144, text_tokens=None, image_tokens=None), cache_creation_input_tokens=0, cache_read_input_tokens=4144))"
       ]
      },
      "execution_count": null,
@@ -851,7 +851,7 @@
     {
      "data": {
       "text/plain": [
-       "PromptTokensDetailsWrapper(audio_tokens=None, cached_tokens=2070, text_tokens=None, image_tokens=None)"
+       "PromptTokensDetailsWrapper(audio_tokens=None, cached_tokens=4144, text_tokens=None, image_tokens=None)"
       ]
      },
      "execution_count": null,
@@ -3833,7 +3833,14 @@
      "text": [
       "Otters are charismatic members of the weasel family found on every continent except Australia and Antarctica. There are 13 species in total, including sea otters and river otters.\n",
       "\n",
-      "These aquatic mammals have elongated bodies, long tails, and soft, dense fur. In fact, otters have the densest fur of any animal—as many as a million hairs per square inch. Webbed feet and powerful tails make otters strong swimmers.\n",
+      "These aquatic mammals have elongated bodies, long tails, and soft, dense fur"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      ". In fact, otters have the densest fur of any animal—as many as a million hairs per square inch. Webbed feet and powerful tails make otters strong swimmers.\n",
       "\n",
       "All otters are expert hunters that eat fish, crustaceans, and other critters. Sea otters float on their backs, place a rock on their chest, then smash mollusks down on it until it breaks open. They're also known for being playful animals, engaging in activities like sliding into water on natural slides."
      ]