Coverage for projects/04-llm-adapter-shadow/src/llm_adapter/runner.py: 95%

41 statements  

« prev     ^ index     » next       coverage.py v7.10.7, created at 2025-09-24 01:32 +0000

1"""Provider runner with fallback handling.""" 

2 

3from __future__ import annotations 

4 

5import time 

6from collections.abc import Sequence 

7from pathlib import Path 

8 

9from .errors import RateLimitError, RetriableError, TimeoutError 

10from .metrics import log_event 

11from .provider_spi import ProviderRequest, ProviderResponse, ProviderSPI 

12from .shadow import DEFAULT_METRICS_PATH, run_with_shadow 

13from .utils import content_hash 

14 

15MetricsPath = str | Path | None 

16 

17 

18class Runner: 

19 """Attempt providers sequentially until one succeeds.""" 

20 

21 def __init__(self, providers: Sequence[ProviderSPI]): 

22 if not providers: 

23 raise ValueError("Runner requires at least one provider") 

24 self.providers: list[ProviderSPI] = list(providers) 

25 

26 def run( 

27 self, 

28 request: ProviderRequest, 

29 shadow: ProviderSPI | None = None, 

30 shadow_metrics_path: MetricsPath = DEFAULT_METRICS_PATH, 

31 ) -> ProviderResponse: 

32 """Execute ``request`` with fallback semantics. 

33 

34 Parameters 

35 ---------- 

36 request: 

37 The prompt/options payload shared across providers. 

38 shadow: 

39 Optional provider that will be executed in the background for 

40 telemetry purposes. 

41 shadow_metrics_path: 

42 JSONL file path for recording metrics. ``None`` disables logging. 

43 """ 

44 

45 last_err: Exception | None = None 

46 metrics_path_str = None if shadow_metrics_path is None else str(Path(shadow_metrics_path)) 

47 request_fingerprint = content_hash( 

48 "runner", request.prompt, request.options, request.max_tokens 

49 ) 

50 

51 def _record_error(err: Exception, attempt: int, provider: ProviderSPI) -> None: 

52 if not metrics_path_str: 

53 return 

54 log_event( 

55 "provider_error", 

56 metrics_path_str, 

57 request_fingerprint=request_fingerprint, 

58 request_hash=content_hash( 

59 provider.name(), request.prompt, request.options, request.max_tokens 

60 ), 

61 provider=provider.name(), 

62 attempt=attempt, 

63 total_providers=len(self.providers), 

64 error_type=type(err).__name__, 

65 error_message=str(err), 

66 ) 

67 

68 for attempt_index, provider in enumerate(self.providers, start=1): 

69 try: 

70 response = run_with_shadow(provider, shadow, request, metrics_path=metrics_path_str) 

71 except RateLimitError as err: 

72 last_err = err 

73 _record_error(err, attempt_index, provider) 

74 time.sleep(0.05) 

75 except (TimeoutError, RetriableError) as err: 

76 last_err = err 

77 _record_error(err, attempt_index, provider) 

78 continue 

79 else: 

80 if metrics_path_str: 

81 log_event( 

82 "provider_success", 

83 metrics_path_str, 

84 request_fingerprint=request_fingerprint, 

85 request_hash=content_hash( 

86 provider.name(), 

87 request.prompt, 

88 request.options, 

89 request.max_tokens, 

90 ), 

91 provider=provider.name(), 

92 attempt=attempt_index, 

93 total_providers=len(self.providers), 

94 latency_ms=response.latency_ms, 

95 shadow_used=shadow is not None, 

96 ) 

97 return response 

98 if metrics_path_str: 

99 log_event( 

100 "provider_chain_failed", 

101 metrics_path_str, 

102 request_fingerprint=request_fingerprint, 

103 provider_attempts=len(self.providers), 

104 providers=[provider.name() for provider in self.providers], 

105 last_error_type=type(last_err).__name__ if last_err else None, 

106 last_error_message=str(last_err) if last_err else None, 

107 ) 

108 raise last_err if last_err is not None else RuntimeError("No providers succeeded") 

109 

110 

111__all__ = ["Runner"]