Files
ARTEX/server/task_llm.go
T
BaaRai e233c02314 Translate all Chinese text to English
Comments, doc comments, UI labels, log and error messages, LLM prompt
strings, markdown docs, test data and assertions. Identifiers, enum
keys, routes, placeholders and code structure are unchanged.

Notes:
- docs/漏洞流量证据.md renamed to docs/vulnerability-traffic-evidence.md
- tests that exercised CJK specific handling (GBK zip names, Chinese
  full text search) now use ASCII equivalents; go build passes and the
  test suite matches the pre translation baseline (remaining failures
  need a PostgreSQL DSN and fail identically on the original)
- frontend mention syntax now uses English kinds (finding, asset, ...)
  matching the backend, chat mentions tests pass
2026-10-08 23:14:34 +02:00

698 lines
26 KiB
Go

package server
import (
"context"
"encoding/json"
"errors"
"fmt"
"iter"
"log"
"path/filepath"
"strconv"
"strings"
"time"
"github.com/Autumn-27/artex/agent"
"github.com/Autumn-27/artex/db"
"github.com/Autumn-27/artex/llmrec"
"github.com/Autumn-27/norma/llm"
"github.com/Autumn-27/norma/transcript"
)
type taskAgentBundle struct {
runtime *taskLLMRuntime // goal decomposition runtime
plannerRuntime *taskLLMRuntime
workerRuntime *taskLLMRuntime
mainRuntime *taskLLMRuntime
pl *agent.Planner
wk *agent.Worker
main *agent.MainAgent
}
type llmAuditProfile struct {
ID int64 `json:"id"`
Name string `json:"name"`
Format string `json:"format"`
Model string `json:"model"`
}
type llmTransitionAudit struct {
Mode string `json:"mode"` // automatic | manual | exhausted
Reason string `json:"reason"`
Previous *llmAuditProfile `json:"previous,omitempty"`
Next *llmAuditProfile `json:"next,omitempty"`
}
type llmActivityMetadata struct {
LLMTransition llmTransitionAudit `json:"llm_transition"`
}
type taskLLMRuntime struct {
s *Server
taskID string
agentKey string
}
type taskLLMError struct {
taskID string
chainExhausted bool
cause error
}
func (e *taskLLMError) Error() string {
if e.chainExhausted {
return fmt.Sprintf("task %s LLM profile chain exhausted: %v", e.taskID, e.cause)
}
return e.cause.Error()
}
func (e *taskLLMError) Unwrap() error { return e.cause }
func isTaskLLMRuntimeError(err error) bool {
var target *taskLLMError
return errors.As(err, &target)
}
func isTaskLLMChainExhausted(err error) bool {
var target *taskLLMError
return errors.As(err, &target) && target.chainExhausted
}
// isQuotaExhaustedError is intentionally strict. A generic 429, auth error,
// network failure, or 5xx does not rotate providers; the response must explicitly
// identify quota, credits, billing balance, or payment exhaustion.
func isQuotaExhaustedError(err error) bool {
return err != nil && agent.IsQuotaExhaustedMessage(err.Error())
}
type taskLLMSelection struct {
task *Task
profileID int64
revision int64
provider llm.Provider
// retry holds the retry parameters resolved for the profile selected this time (profile override → global policy → built-in default).
// Same-provider safe-window retries follow it, so switching profiles switches the retry cadence.
retry agent.RetryConfig
}
type taskLLMStreamHooks struct {
current func() (taskLLMSelection, error)
exhaust func(taskLLMSelection, error) (db.TaskLLMTransition, error)
transition func(taskLLMSelection, db.TaskLLMTransition, error)
}
// current resolves the provider this role runs on, by precedence:
// Agent binding → task LLM profile chain → global/environment configuration.
// Binding wins over the task chain: a role explicitly assigned a model keeps running
// on that model; only when the binding is missing or fails to build does it fall back
// to the task chain, and to the global configuration when the chain is empty.
// The returned profile id is non-zero only on the task-chain path — streamTaskLLM uses it to decide whether a quota error should advance the task's failover state (binding/global paths leave the chain state untouched, preserving existing semantics).
func (r *taskLLMRuntime) current() (taskLLMSelection, error) {
taskNum, err := parseTaskID(r.taskID)
if err != nil {
return taskLLMSelection{}, err
}
pt, err := r.s.m.pg.GetTask(taskNum)
if err != nil {
return taskLLMSelection{}, err
}
if pt == nil {
return taskLLMSelection{}, fmt.Errorf("task %s not found", r.taskID)
}
r.s.syncTaskLLMState(pt)
t, _ := r.s.m.Task(r.taskID)
sel := taskLLMSelection{task: t, revision: pt.LLMChainRevision}
if prov, cfg, ok := r.s.agentBindingProvider(r.agentKey); ok {
sel.provider, sel.retry = prov, cfg.Retry
return sel, nil
}
if len(pt.LLMProfileIDs) > 0 {
if pt.ActiveLLMProfileID == nil {
return sel, &taskLLMError{taskID: r.taskID, chainExhausted: true, cause: errors.New("all selected profiles are quota exhausted")}
}
sel.profileID = *pt.ActiveLLMProfileID
prov, cfg, ok := r.s.providerForProfile(sel.profileID)
if !ok {
return sel, fmt.Errorf("LLM profile #%d is missing or invalid", sel.profileID)
}
sel.provider, sel.retry = prov, cfg.Retry
return sel, nil
}
prov, cfg, ok := r.s.globalProvider()
if !ok {
return sel, fmt.Errorf("task %s has no available fallback LLM provider", r.taskID)
}
sel.provider, sel.retry = prov, cfg.Retry
return sel, nil
}
// activeCfg resolves the task's currently-active LLM config, mirroring current()'s
// source precedence (agent binding → active chain profile → global). Read-only and
// best-effort: ok=false when nothing resolves, leaving the per-setting fallback to
// the caller. If failover switches profiles, the change takes effect on the next
// agent run (a fresh Session is built per run in captureRun).
func (r *taskLLMRuntime) activeCfg() (agent.Config, bool) {
taskNum, err := parseTaskID(r.taskID)
if err != nil {
return agent.Config{}, false
}
pt, err := r.s.m.pg.GetTask(taskNum)
if err != nil || pt == nil {
return agent.Config{}, false
}
if _, cfg, ok := r.s.agentBindingProvider(r.agentKey); ok {
return cfg, true
}
if len(pt.LLMProfileIDs) > 0 && pt.ActiveLLMProfileID != nil {
if _, cfg, ok := r.s.providerForProfile(*pt.ActiveLLMProfileID); ok {
return cfg, true
}
}
if _, cfg, ok := r.s.globalProvider(); ok {
return cfg, true
}
return agent.Config{}, false
}
// nonStreaming reports whether the task's currently-active LLM source is set to
// non-streaming. Unresolvable → streaming (false), the safe default.
func (r *taskLLMRuntime) nonStreaming() bool {
cfg, ok := r.activeCfg()
return ok && !cfg.Stream
}
// maxTokens returns the currently-active source's per-reply output cap.
// Unresolvable → 0, i.e. send no cap, matching the pre-setting behaviour.
func (r *taskLLMRuntime) maxTokens() int {
cfg, _ := r.activeCfg() // zero value 0 when no configuration resolves
return cfg.MaxTokens
}
func parseTaskID(id string) (int64, error) {
n, err := strconv.ParseInt(id, 10, 64)
if err != nil || n <= 0 {
return 0, fmt.Errorf("invalid task id %q", id)
}
return n, nil
}
// streamHooks builds the failover/exhaustion callbacks shared by Stream and
// Complete: how to read the current profile selection, how to mark it quota
// exhausted, and how to emit a failover transition.
func (r *taskLLMRuntime) streamHooks() taskLLMStreamHooks {
return taskLLMStreamHooks{
current: r.current,
exhaust: func(selection taskLLMSelection, cause error) (db.TaskLLMTransition, error) {
taskNum, _ := parseTaskID(r.taskID)
transition, err := r.s.m.pg.MarkTaskLLMProfileQuotaExhaustedAtRevision(taskNum, selection.profileID, selection.revision, cause.Error())
if err != nil {
return transition, err
}
if pt, getErr := r.s.m.pg.GetTask(taskNum); getErr == nil && pt != nil {
r.s.syncTaskLLMState(pt)
}
return transition, nil
},
transition: func(selection taskLLMSelection, transition db.TaskLLMTransition, cause error) {
r.s.emitTaskLLMTransition(selection.task, transition, cause)
},
}
}
func (r *taskLLMRuntime) Stream(ctx context.Context, req llm.CompletionRequest) iter.Seq2[llm.StreamEvent, error] {
ctx = llmrec.WithTaskID(ctx, r.taskID)
return streamTaskLLM(ctx, r.taskID, req, r.streamHooks())
}
// Complete is the non-streaming counterpart of Stream. A non-streaming call is
// atomic — it never delivers partial output — so every failure is safe to retry
// on the same provider or fail over to the next profile without risking
// duplicated model output or tool execution (the "committed" bookkeeping the
// streaming path needs is unnecessary here).
func (r *taskLLMRuntime) Complete(ctx context.Context, req llm.CompletionRequest) (llm.Message, string, llm.Usage, error) {
ctx = llmrec.WithTaskID(ctx, r.taskID)
return completeTaskLLM(ctx, r.taskID, req, r.streamHooks())
}
func completeTaskLLM(ctx context.Context, taskID string, req llm.CompletionRequest, hooks taskLLMStreamHooks) (llm.Message, string, llm.Usage, error) {
for {
selection, err := hooks.current()
if err != nil {
return llm.Message{}, "", llm.Usage{}, err
}
var (
msg llm.Message
sr string
usage llm.Usage
callErr error
)
// Same-provider safe-window retry: a non-streaming call either succeeds or
// fails as a whole — no partially delivered output — so any transient failure can be retried verbatim.
retries, backoffOf := sameProviderRetryPolicy(selection.retry)
for attempt := 0; ; attempt++ {
msg, sr, usage, callErr = selection.provider.Complete(ctx, req)
if callErr != nil && ctx.Err() == nil &&
attempt < retries && isRetryableStreamError(callErr) {
backoff := backoffOf(attempt)
log.Printf("[task-llm] task %s non-streaming call failed, retrying on the same provider after %v (%d/%d): %v",
taskID, backoff, attempt+1, retries, callErr)
if sleepCtx(ctx, backoff) {
break // ctx cancelled during backoff → stop retrying
}
continue
}
break
}
if callErr == nil {
return msg, sr, usage, nil
}
// profileID=0: the explicit chain was cleared; non-quota error: pass through. Neither changes the task failover state.
if selection.profileID == 0 || !isQuotaExhaustedError(callErr) {
return llm.Message{}, "", llm.Usage{}, callErr
}
transition, markErr := hooks.exhaust(selection, callErr)
if markErr != nil {
return llm.Message{}, "", llm.Usage{}, fmt.Errorf("mark profile quota exhausted after %v: %w", callErr, markErr)
}
if transition.Advanced && !transition.Stale && hooks.transition != nil {
hooks.transition(selection, transition, callErr)
}
if !transition.Stale && transition.NextProfileID == nil {
return llm.Message{}, "", llm.Usage{}, &taskLLMError{taskID: taskID, chainExhausted: transition.ChainExhausted, cause: callErr}
}
// No output has been delivered to the caller, so replaying the same logical request on the next profile is safe.
}
}
func streamTaskLLM(ctx context.Context, taskID string, req llm.CompletionRequest, hooks taskLLMStreamHooks) iter.Seq2[llm.StreamEvent, error] {
return func(yield func(llm.StreamEvent, error) bool) {
for {
selection, err := hooks.current()
if err != nil {
yield(llm.StreamEvent{}, err)
return
}
committed := false
var pending []llm.StreamEvent
var streamErr error
retries, backoffOf := sameProviderRetryPolicy(selection.retry)
// Same-provider safe-window retry: a transient failure before committed
// (nothing delivered to the caller yet) can be replayed verbatim without
// duplicating model output or tool executions. After committed, on ctx cancellation, or on deterministic/quota errors it breaks out to the original pass-through/failover logic below.
for attempt := 0; ; attempt++ {
committed = false
pending = nil
streamErr = nil
for event, err := range selection.provider.Stream(ctx, req) {
if err != nil {
streamErr = err
break
}
if !committed && !streamEventCommitsOutput(event) {
pending = append(pending, event)
continue
}
if !committed {
for _, buffered := range pending {
if !yield(buffered, nil) {
return
}
}
pending = nil
committed = true
}
if !yield(event, nil) {
return
}
}
if streamErr != nil && !committed && ctx.Err() == nil &&
attempt < retries && isRetryableStreamError(streamErr) {
backoff := backoffOf(attempt)
log.Printf("[task-llm] task %s pre-commit stream failure, retrying on the same provider after %v (%d/%d): %v",
taskID, backoff, attempt+1, retries, streamErr)
if sleepCtx(ctx, backoff) {
break // ctx cancelled during backoff → stop retrying
}
continue
}
break
}
if streamErr == nil {
for _, buffered := range pending {
if !yield(buffered, nil) {
return
}
}
return
}
// profileID=0 means the explicit chain was cleared while this stable task
// bundle was still in use. Agent/global fallback errors follow the legacy
// behavior and never mutate task failover state.
if selection.profileID == 0 || !isQuotaExhaustedError(streamErr) {
for _, buffered := range pending {
if !yield(buffered, nil) {
return
}
}
yield(llm.StreamEvent{}, streamErr)
return
}
transition, markErr := hooks.exhaust(selection, streamErr)
if markErr != nil {
cause := fmt.Errorf("mark profile quota exhausted after %v: %w", streamErr, markErr)
if committed {
// Output may already have driven tool execution. Report the persistence
// failure, but classify it as router-handled so the worker does not
// replay the entire intent and duplicate those side effects.
yield(llm.StreamEvent{}, &taskLLMError{taskID: taskID, cause: cause})
} else {
yield(llm.StreamEvent{}, cause)
}
return
}
if transition.Advanced && !transition.Stale && hooks.transition != nil {
hooks.transition(selection, transition, streamErr)
}
if committed || (!transition.Stale && transition.NextProfileID == nil) {
yield(llm.StreamEvent{}, &taskLLMError{taskID: taskID, chainExhausted: transition.ChainExhausted, cause: streamErr})
return
}
// No event reached the caller, so replaying the same logical request on
// the next profile cannot duplicate model output or tool execution.
}
}
}
func streamEventCommitsOutput(event llm.StreamEvent) bool {
switch event.Type {
case llm.SETextDelta, llm.SEThinkingDelta, llm.SEToolInputJSON:
return event.Text != ""
case llm.SEToolUseStart, llm.SEMessageDelta, llm.SEMessageStop:
return true
default:
return false
}
}
// The [default] retry count against the same provider within the pre-commit safe
// window. The SDK's doStream retries only the connection phase (before the 200); once
// the stream starts, mid-stream drops / overloaded / in-stream 429 and other transient
// faults bubble straight up as model_error with zero retries. As long as not one token has been handed to the caller (!committed), replaying the exact same request duplicates no model output or tool side effects, so this adds a same-provider backoff retry layer to absorb such jitter before the whole intent re-runs.
// Overridable by an LLM profile's retry overrides / the global retry policy.
const sameProviderStreamRetries = 2
// sameProviderRetryBackoff is the [default] backoff before retry attempt #attempt
// (0.5s, 1s…, capped at 4s) — same style as the SDK's exponential ladder but with a
// smaller cap so worker wrap-up/cancellation responsiveness is not dragged out. Exposed as variables so tests can zero the backoff.
var sameProviderRetryBackoff = func(attempt int) time.Duration {
return min(500*time.Millisecond*(1<<attempt), 4*time.Second)
}
// sameProviderRetryPolicy resolves which same-provider retry parameters this call
// uses: a configured count wins (negative = turn this retry layer off), a configured
// interval swaps exponential backoff for a fixed interval, and with neither set it stays byte-for-byte identical to before it was configurable.
func sameProviderRetryPolicy(r agent.RetryConfig) (retries int, backoff func(int) time.Duration) {
retries, backoff = sameProviderStreamRetries, sameProviderRetryBackoff
if r.StreamAttempts != 0 {
retries = max(r.StreamAttempts, 0)
}
if r.StreamInterval > 0 {
d := r.StreamInterval
backoff = func(int) time.Duration { return d }
}
return retries, backoff
}
// isRetryableStreamError decides whether a "pre-commit stream failure" is worth replaying on the same provider.
// Transient transport breaks / provider overload / rate limiting recover on their own and are safe to retry; these three are not:
// - quota exhausted: left to profile failover, don't burn retries here in vain
// - context too long: replaying the same request is useless; the harness's reactive compaction is the backstop
// - deterministic 4xx rejection (400/401/403/404/422): every provider would fail the same way
func isRetryableStreamError(err error) bool {
if err == nil {
return false
}
if isQuotaExhaustedError(err) {
return false
}
s := strings.ToLower(err.Error())
if strings.Contains(s, "too long") || strings.Contains(s, "context length") ||
strings.Contains(s, "context_length") || strings.Contains(s, "maximum context") ||
strings.Contains(s, "status 413") {
return false
}
for _, code := range []string{"status 400", "status 401", "status 403", "status 404", "status 422"} {
if strings.Contains(s, code) {
return false
}
}
// Everything else (transport reset/EOF/timeout, 408/429/5xx, in-stream error events
// such as anthropic overloaded_error, etc.) is treated as transient and retryable.
return true
}
// CompactionWindow mirrors current()'s precedence so the context window always
// matches the provider the role will actually stream on.
func (r *taskLLMRuntime) CompactionWindow() int {
if _, cfg, ok := r.s.agentBindingProvider(r.agentKey); ok {
return cfg.CompactionWindow()
}
taskNum, err := parseTaskID(r.taskID)
if err != nil {
return (agent.Config{}).CompactionWindow()
}
chain, err := r.s.m.pg.TaskLLMProfiles(taskNum)
if err != nil {
return (agent.Config{}).CompactionWindow()
}
if len(chain) == 0 {
if _, cfg, ok := r.s.globalProvider(); ok {
return cfg.CompactionWindow()
}
return (agent.Config{}).CompactionWindow()
}
minimum := 0
for _, entry := range chain {
if cfg, ok := r.s.loadProfileConfig(entry.ProfileID); ok {
window := cfg.CompactionWindow()
if minimum == 0 || window < minimum {
minimum = window
}
}
}
if minimum == 0 {
return (agent.Config{}).CompactionWindow()
}
return minimum
}
// agentBindingProvider resolves the profile a role is explicitly bound to
// (agents.llm_profile_id) — the highest-precedence level for task agents. ok=false
// when the role has no binding or the bound profile no longer builds, so callers
// fall through to the task chain. A bound profile stays exclusive unless
// llm_pool_bind_fallback is on, which is what poolForBinding encodes.
func (s *Server) agentBindingProvider(agentKey string) (llm.Provider, agent.Config, bool) {
id := s.effectiveProfileForAgent(agentKey, nil)
if id == nil {
return nil, agent.Config{}, false
}
prov, cfg, ok := s.providerForProfile(*id)
if !ok {
return nil, agent.Config{}, false
}
return s.poolForBinding(*id, prov, cfg), cfg, true
}
// globalProvider returns the process-wide provider (persisted active profile or
// environment config) — the last resort once a role has neither a binding nor a
// task chain.
func (s *Server) globalProvider() (llm.Provider, agent.Config, bool) {
s.cfgMu.Lock()
defer s.cfgMu.Unlock()
if !s.llmOn || s.llmProv == nil {
return nil, agent.Config{}, false
}
return s.llmProv, s.llmCfg, true
}
// taskRuntimeAvailable reports whether every listed role can resolve a provider
// under the runtime precedence in current(): the role's own binding first, then
// the task chain, then global. An exhausted chain is a hard stop for unbound
// roles rather than a silent fall through to global — same as current().
func (s *Server) taskRuntimeAvailable(t *Task, agentKeys ...string) bool {
if t == nil || len(agentKeys) == 0 {
return false
}
state := t.llmStateSnapshot()
unboundReady := false
if len(state.ProfileIDs) > 0 {
if state.ActiveID != nil {
_, _, unboundReady = s.providerForProfile(*state.ActiveID)
}
} else {
_, _, unboundReady = s.globalProvider()
}
for _, key := range agentKeys {
if _, _, ok := s.agentBindingProvider(key); ok {
continue
}
if !unboundReady {
return false
}
}
return true
}
func (s *Server) invalidateTaskAgents() {
s.taskAgentMu.Lock()
s.taskAgents = map[string]*taskAgentBundle{}
s.taskAgentMu.Unlock()
}
func (s *Server) agentsForTask(t *Task) *taskAgentBundle {
s.taskAgentMu.Lock()
defer s.taskAgentMu.Unlock()
if bundle := s.taskAgents[t.ID]; bundle != nil {
return bundle
}
goalRuntime := &taskLLMRuntime{s: s, taskID: t.ID, agentKey: "goals"}
plannerRuntime := &taskLLMRuntime{s: s, taskID: t.ID, agentKey: "planner"}
workerRuntime := &taskLLMRuntime{s: s, taskID: t.ID, agentKey: "worker"}
mainRuntime := &taskLLMRuntime{s: s, taskID: t.ID, agentKey: "mainagent"}
tx := transcript.NewStore(filepath.Join(s.m.dir, "transcripts"))
window := workerRuntime.CompactionWindow()
wk := agent.NewWorker(workerRuntime, "task-router", s.m.dir, tx, window, s.agentMaxTurns("worker"))
wk.SetFindingRecorder(s.evidenceStore())
wk.SetCompactionWindowResolver(workerRuntime.CompactionWindow)
wk.SetNonStreaming(workerRuntime.nonStreaming) // per the task's currently active profile streaming switch (read every turn)
wk.SetMaxTokens(workerRuntime.maxTokens) // as above, the output cap also follows the currently active profile
wk.SetNoaEnabled(s.m.NoaCompactionEnabled) // experimental: noa context compaction (platform-level switch, read every run)
wk.SetRunTimeout(time.Duration(s.agentRunSeconds("worker")) * time.Second)
wk.SetProxy(s.m.ProxyAddr(), s.m.ProxyCACert())
wk.SetWebSearch(s.webSearchFor("worker"))
wk.SetConstraintInject(s.constraintInjectWorker) // operation-constraint injection into the worker (configurable, on by default)
pl := agent.NewPlanner(plannerRuntime, "task-router", s.m.dir, tx, plannerRuntime.CompactionWindow(), s.agentMaxTurns("planner"))
pl.SetFindingRecorder(s.evidenceStore())
pl.SetCompactionWindowResolver(plannerRuntime.CompactionWindow)
pl.SetNonStreaming(plannerRuntime.nonStreaming)
pl.SetMaxTokens(plannerRuntime.maxTokens)
pl.SetNoaEnabled(s.m.NoaCompactionEnabled) // experimental: noa context compaction (platform-level switch, read every run)
pl.SetKillWork(s.engine.KillWork)
pl.SetSteerWork(s.engine.SteerWork)
pl.SetProxy(s.m.ProxyAddr(), s.m.ProxyCACert())
pl.SetWebSearch(s.webSearchFor("planner"))
pl.SetConstraintInject(s.constraintInjectPlanner) // operation-constraint injection into the planner (configurable, on by default)
// cold-digest §7: background compaction of cold nodes. This per-task planner set
// (agentsForTask) is exactly what the engine actually drives through the authoritative
// resolver, so the Compactor must hook in here. The planner provider goes through task routing (§4: same model as the agent, resolved with the task's LLM chain); compaction uses Complete to generate the body in one shot.
pl.SetCompactor(agent.NewCompactor(plannerRuntime, "task-router"))
main := agent.NewMainAgent(mainRuntime, "task-router", s.m.dir, tx, mainRuntime.CompactionWindow(), s.agentMaxTurns("mainagent"))
main.SetFindingRecorder(s.evidenceStore())
main.SetCompactionWindowResolver(mainRuntime.CompactionWindow)
main.SetNonStreaming(mainRuntime.nonStreaming)
main.SetMaxTokens(mainRuntime.maxTokens)
main.SetNoaEnabled(s.m.NoaCompactionEnabled) // experimental: noa context compaction (platform-level switch, read every run)
main.SetProxy(s.m.ProxyAddr(), s.m.ProxyCACert())
main.SetWebSearch(s.webSearchFor("mainagent"))
main.SetSteerWork(s.engine.SteerWork) // steer_work: real-time course correction on running work by a human
bundle := &taskAgentBundle{
runtime: goalRuntime, plannerRuntime: plannerRuntime, workerRuntime: workerRuntime,
mainRuntime: mainRuntime, pl: pl, wk: wk, main: main,
}
s.taskAgents[t.ID] = bundle
return bundle
}
func (s *Server) syncTaskLLMState(pt *db.Task) {
if pt == nil {
return
}
id := fmt.Sprintf("%d", pt.ID)
s.m.mu.Lock()
if task := s.m.tasks[id]; task != nil {
task.setLLMState(pt.LLMProfileID, pt.ActiveLLMProfileID, pt.LLMProfileIDs, pt.LLMChainRevision, pt.LLMFailoverState, pt.LLMFailoverReason)
}
s.m.mu.Unlock()
}
func (s *Server) emitTaskLLMTransition(t *Task, transition db.TaskLLMTransition, cause error) {
if t == nil {
return
}
previous := s.llmAuditProfile(transition.PreviousProfileID)
var next *llmAuditProfile
if transition.NextProfileID != nil {
next = s.llmAuditProfile(*transition.NextProfileID)
}
mode := "automatic"
kind := "llm_switch"
summary := fmt.Sprintf("%s quota exhausted", llmAuditProfileLabel(previous))
if transition.NextProfileID != nil {
summary += fmt.Sprintf(", subsequent calls switched to %s", llmAuditProfileLabel(next))
} else {
mode = "exhausted"
kind = "llm_failover"
summary += ", profile chain exhausted"
}
metadata, _ := json.Marshal(llmActivityMetadata{LLMTransition: llmTransitionAudit{
Mode: mode, Reason: cause.Error(), Previous: previous, Next: next,
}})
s.engine.emitActivity(t, db.Activity{Worker: "system", Kind: kind, IsError: transition.ChainExhausted, Summary: summary, Detail: cause.Error(), Metadata: metadata})
log.Printf("[llm-failover] task %s: %s", t.ID, summary)
}
func (s *Server) llmAuditProfile(id int64) *llmAuditProfile {
if id <= 0 || s.m == nil || s.m.pg == nil {
return nil
}
p, err := s.m.pg.ProfileByID(id)
if err != nil || p == nil {
return &llmAuditProfile{ID: id, Name: fmt.Sprintf("profile #%d", id)}
}
return &llmAuditProfile{ID: p.ID, Name: p.Name, Format: p.Format, Model: p.Model}
}
func llmAuditProfileLabel(profile *llmAuditProfile) string {
if profile == nil {
return "default profile"
}
name := profile.Name
if name == "" {
name = fmt.Sprintf("profile #%d", profile.ID)
}
detail := []string{}
if profile.Format != "" {
detail = append(detail, profile.Format)
}
if profile.Model != "" {
detail = append(detail, profile.Model)
}
if len(detail) == 0 {
return name
}
return fmt.Sprintf("%s (%s)", name, strings.Join(detail, " / "))
}
func sameOptionalID(a, b *int64) bool {
if a == nil || b == nil {
return a == nil && b == nil
}
return *a == *b
}
func (s *Server) emitManualTaskLLMSwitch(t *Task, previousID, nextID *int64) db.Activity {
previous := (*llmAuditProfile)(nil)
next := (*llmAuditProfile)(nil)
if previousID != nil {
previous = s.llmAuditProfile(*previousID)
}
if nextID != nil {
next = s.llmAuditProfile(*nextID)
}
summary := fmt.Sprintf("user manually switched the task LLM from %s to %s", llmAuditProfileLabel(previous), llmAuditProfileLabel(next))
metadata, _ := json.Marshal(llmActivityMetadata{LLMTransition: llmTransitionAudit{
Mode: "manual", Reason: "user manually switched the task LLM", Previous: previous, Next: next,
}})
return s.engine.emitActivity(t, db.Activity{Worker: "system", Kind: "llm_switch", Summary: summary, Detail: summary, Metadata: metadata})
}