diff --git a/vars/main.yml b/vars/main.yml index d649151..2b13743 100644 --- a/vars/main.yml +++ b/vars/main.yml @@ -94,15 +94,28 @@ litellm_models: backend: ollama_chat/llama3.1:8b api_base: "{{ anvil_ollama_base }}" api_key: ollama-no-auth - # The structured-output tier, added 2026-09-12. llama3.1:8b answers fast but - # will not hold strict JSON: paperless-ai got "Expected ',' or '}' after - # property value in JSON at position 263" on a 1,921-token reply. The Kestra - # flows escape this by setting response_format={"type":"json_object"}; - # paperless-ai does not, so it needs a model that keeps the shape unprompted. - # 14B is the smallest that does, and at ~9 GB it co-exists with anything here. + # The structured-output tier for paperless-ai, added 2026-09-12. ~9 GB, so it + # co-exists with anything else here. + # + # ⚠️ NOTE THE PROVIDER: `openai/` with an explicit /v1, NOT `ollama_chat/` + # like every other local_* route above. This is deliberate and measured. + # + # paperless-ai kept failing with "Invalid JSON response from API" even after + # the real cause of the big failures (TOKEN_LIMIT=128000 against a 16386 + # window) was fixed and requests dropped from 44k to 2.1k tokens. Isolation + # on 2026-09-12, replaying paperless-ai's exact system prompt 3x per path: + # Anvil /v1/chat/completions (OpenAI-compat) → 3/3 VALID JSON + # Anvil /api/chat (native) → 3/3 VALID JSON + # via LiteLLM ollama_chat/ → invalid every time + # Model, prompt, temperature and context were identical across all three, so + # the fault is the provider's transformation, not the model. + # + # `openai/` is a near-passthrough to the endpoint that was proven to work. + # ⛔ Do not "tidy" this back to ollama_chat/ for consistency with its + # neighbours. The inconsistency is the fix. - name: local_qwen2.5-14b - backend: ollama_chat/qwen2.5:14b - api_base: "{{ anvil_ollama_base }}" + backend: openai/qwen2.5:14b + api_base: "{{ anvil_ollama_base }}/v1" api_key: ollama-no-auth # Shadow alias SOLELY for paperless-ai's hardcoded gpt-4o-mini validation # probe (see the "Narrow exception" note above). Routed to free local