f09ae248f9
* add staging ios app build script * feat(api): add OpenTelemetry LGTM stack for local observability Wire OTel traces, metrics, and logs through a collector to Tempo, Prometheus, and Loki, with custom LLM instrumentation, dev compose overlay, Grafana provisioning, env templates, and development.sh auto-start. * feat: expand observability, harden dev tooling, and fix expo staging UX Add business and LLM Prometheus metrics with Grafana dashboards, alerting, and a metrics verification script. Wire telemetry through adapters and core LLM paths, and document the local LGTM workflow. Fix development.sh for macOS bash 3.2, open Grafana and eval-web in Chrome, and repair eval-web auto-open (unbound EVAL_WEB_BROWSER_SCHEDULED). Merge internal-eval into the main dev script with improved compose handling. Require EXPO_PUBLIC_* at build time, improve iOS HTTP ATS for staging IPs, show memoir empty state instead of load errors when no chapters exist, and add jest env setup plus chapter list response normalization. * chore: enable Grafana Assistant Cursor plugin * fix: memoir empty state and repair withdrawn 0020_chapters_book_id stamp Show empty memoir UI when the chapter list succeeds with no items; treat auth/404 as non-fatal. Extend alembic revision repair so local dev DBs stamped with the removed 0020_chapters_book_id migration can roll back and upgrade to 0019. --------- Co-authored-by: Kevin <kevin@brighteng.org> Co-authored-by: Cursor <cursoragent@cursor.com>
119 lines
4.1 KiB
Python
119 lines
4.1 KiB
Python
"""LLM telemetry helpers (no real Collector required)."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from types import SimpleNamespace
|
|
from unittest.mock import patch
|
|
|
|
import pytest
|
|
|
|
from app.core import llm_telemetry
|
|
from app.core.config import settings
|
|
|
|
|
|
class TestExtractTokenUsage:
|
|
def test_usage_metadata_object(self) -> None:
|
|
msg = SimpleNamespace(usage_metadata=SimpleNamespace(input_tokens=10, output_tokens=4))
|
|
assert llm_telemetry.extract_token_usage(msg) == (10, 4)
|
|
|
|
def test_response_metadata_dict(self) -> None:
|
|
msg = SimpleNamespace(
|
|
usage_metadata=None,
|
|
response_metadata={"token_usage": {"prompt_tokens": 3, "completion_tokens": 7}},
|
|
)
|
|
assert llm_telemetry.extract_token_usage(msg) == (3, 7)
|
|
|
|
def test_missing_usage_returns_zero(self) -> None:
|
|
assert llm_telemetry.extract_token_usage(SimpleNamespace()) == (0, 0)
|
|
|
|
|
|
class TestOtelDisabledNoOp:
|
|
def test_record_llm_completion_disabled(self, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
monkeypatch.setattr(settings, "otel_enabled", False)
|
|
llm_telemetry.record_llm_completion(
|
|
agent="Test",
|
|
provider="mock",
|
|
model="m",
|
|
duration_ms=1.0,
|
|
input_tokens=5,
|
|
output_tokens=2,
|
|
)
|
|
|
|
def test_langchain_invoke_span_disabled(self, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
monkeypatch.setattr(settings, "otel_enabled", False)
|
|
with llm_telemetry.langchain_invoke_span(
|
|
agent="Test",
|
|
provider="mock",
|
|
model="m",
|
|
call_type="chat",
|
|
) as ctx:
|
|
ctx["response"] = SimpleNamespace(
|
|
usage_metadata=SimpleNamespace(input_tokens=1, output_tokens=1)
|
|
)
|
|
assert ctx["outcome"] == "ok"
|
|
|
|
|
|
class TestLangchainInvokeSpanRecordsTokens:
|
|
def test_records_completion_with_tokens(self, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
monkeypatch.setattr(settings, "otel_enabled", True)
|
|
recorded: list[dict] = []
|
|
|
|
def _capture(**kwargs: object) -> None:
|
|
recorded.append(kwargs)
|
|
|
|
with patch.object(llm_telemetry, "record_llm_completion", side_effect=_capture):
|
|
with llm_telemetry.langchain_invoke_span(
|
|
agent="TestAgent",
|
|
provider="mock",
|
|
model="m1",
|
|
call_type="chat",
|
|
) as ctx:
|
|
ctx["response"] = SimpleNamespace(
|
|
usage_metadata=SimpleNamespace(input_tokens=11, output_tokens=5)
|
|
)
|
|
|
|
assert len(recorded) == 1
|
|
assert recorded[0]["input_tokens"] == 11
|
|
assert recorded[0]["output_tokens"] == 5
|
|
assert recorded[0]["agent"] == "TestAgent"
|
|
|
|
|
|
class TestObserveAinvokeExtraAttributes:
|
|
@pytest.mark.asyncio
|
|
async def test_response_latency_on_span(self, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
monkeypatch.setattr(settings, "otel_enabled", True)
|
|
from opentelemetry.sdk.trace import TracerProvider
|
|
from opentelemetry.sdk.trace.export import SimpleSpanProcessor
|
|
from opentelemetry.sdk.trace.export.in_memory_span_exporter import (
|
|
InMemorySpanExporter,
|
|
)
|
|
from opentelemetry import trace
|
|
|
|
exporter = InMemorySpanExporter()
|
|
provider = TracerProvider()
|
|
provider.add_span_processor(SimpleSpanProcessor(exporter))
|
|
monkeypatch.setattr(
|
|
"app.core.llm_telemetry.get_tracer",
|
|
lambda _name: provider.get_tracer("test"),
|
|
)
|
|
|
|
class _LLM:
|
|
async def ainvoke(self, messages: list) -> SimpleNamespace:
|
|
return SimpleNamespace(
|
|
usage_metadata=SimpleNamespace(input_tokens=1, output_tokens=1)
|
|
)
|
|
|
|
await llm_telemetry.observe_ainvoke(
|
|
_LLM(),
|
|
[],
|
|
agent="Test",
|
|
provider="mock",
|
|
model="m",
|
|
extra_span_attributes={"llm.custom": "x"},
|
|
)
|
|
spans = exporter.get_finished_spans()
|
|
assert spans
|
|
attrs = dict(spans[-1].attributes or {})
|
|
assert "llm.response_latency_ms" in attrs
|
|
assert attrs.get("llm.custom") == "x"
|