chat: Add optional LLM transcript directory

config/settings/base.py

@@ -255,3 +255,8 @@
 DATA_UPLOAD_MAX_NUMBER_FIELDS = None
 
 WAGTAILEMBEDS_FINDERS = [{'class': 'mpicms.base.oembed'}]
+
+# CHAT
+# Directory for transcripts of the LLM communication, one file per request.
+# Transcripts are not written if None. Old files are not removed automatically.
+CHAT_TRANSCRIPT_DIR = None

mpicms/chat/services.py

@@ -8,7 +8,8 @@
 
 import logging
 import os
-from datetime import date
+import time
+from datetime import date, datetime
 
 import html2text as _html2text
 from django.conf import settings
@@ -336,6 +337,41 @@ def retrieve(query: str, lang: str):
 }
 
 
+def _write_transcript(started, duration, lang, messages, response, exc):
+    """Write the LLM request and response to a file in CHAT_TRANSCRIPT_DIR."""
+    directory = getattr(settings, 'CHAT_TRANSCRIPT_DIR', None)
+    if not directory:
+        return
+    lines = [
+        f"time:     {started.isoformat(timespec='milliseconds')}",
+        f"pid:      {os.getpid()}",
+        f"model:    {CHAT_MODEL}",
+        f"lang:     {lang}",
+        f"duration: {duration:.1f} s",
+    ]
+    if exc is not None:
+        lines.append(f"error:    {type(exc).__name__}: {exc}")
+    else:
+        choice = response.choices[0]
+        lines.append(f"response model: {response.model}")
+        lines.append(f"finish_reason:  {choice.finish_reason}")
+        if response.usage:
+            lines.append(f"tokens:   prompt={response.usage.prompt_tokens} "
+                         f"completion={response.usage.completion_tokens} "
+                         f"total={response.usage.total_tokens}")
+    for i, m in enumerate(messages):
+        lines += ['', f"======== request message {i}: {m['role']} ========", m['content']]
+    if exc is None:
+        lines += ['', f"======== response: {choice.message.role} ========", choice.message.content or '']
+    path = os.path.join(directory, f"{started:%Y%m%dT%H%M%S.%f}-{os.getpid()}.txt")
+    try:
+        fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
+        with open(fd, 'w', encoding='utf-8') as f:
+            f.write('\n'.join(lines) + '\n')
+    except OSError as e:
+        logger.warning("cannot write chat transcript %s: %s", path, e)
+
+
 def ask(query: str, history: list, lang: str, page_stack: list, opts={}) -> tuple:
     """Return (answer, updated_page_stack).
 
@@ -402,6 +438,8 @@ def ask(query: str, history: list, lang: str, page_stack: list, opts={}) -> tupl
         from pprint import pprint
         pprint(messages)
 
+    started = datetime.now().astimezone()
+    t0 = time.monotonic()
     try:
         response = get_chat_client().chat.completions.create(
             model=CHAT_MODEL,
@@ -410,9 +448,11 @@ def ask(query: str, history: list, lang: str, page_stack: list, opts={}) -> tupl
             top_p=0.01,
         )
     except Exception as exc:
+        _write_transcript(started, time.monotonic() - t0, lang, messages, None, exc)
         if lang == 'en':
             return f"The assistant is not responding. Please try again later. ({type(exc).__name__})", updated_stack
         return f"Der Assistent antwortet nicht. Bitte versuche es später erneut. ({type(exc).__name__})", updated_stack
+    _write_transcript(started, time.monotonic() - t0, lang, messages, response, None)
     if response.usage:
         logger.debug("LLM tokens: prompt=%d completion=%d total=%d",
                      response.usage.prompt_tokens,