fix: test isolation for the escalation streak + compliant goal fixtures

- resetLLMFailStreaksForTest helper: the package-global streak is in-memory
  state, so the no-switch-on-ordinary-error subtests now reset it before each
  case (accumulation across subtests sharing task/profile keys made the third
  case escalate, breaking the contract the test guards)
- core lifecycle and credentials fixtures: goals now name the lab target, per
  the ambiguity gate (these tests exercise lifecycle/credentials, not the gate)
- canonical reset helper moved into task_llm.go next to the counter it resets
This commit is contained in:
BaaRai
2026-10-21 15:08:37 +02:00
parent b237fbaa32
commit 6894c391dd
5 changed files with 13 additions and 9 deletions
+1 -1
View File
@@ -62,7 +62,7 @@ func TestCoreTaskLifecyclePG(t *testing.T) {
}
// create a task → 201, returns PG task (string id + exploration_id)
code, out := doRetry("POST", "/api/tasks", map[string]string{"description": "smoke", "goal": "test SQLi/XSS"})
code, out := doRetry("POST", "/api/tasks", map[string]string{"description": "smoke", "goal": "Probe the lab target at http://127.0.0.1:9/v1 for unauthenticated data exposure; record one record as evidence and stop."})
if code != 201 {
t.Fatalf("create task: %d (%v)", code, out)
}
+2 -2
View File
@@ -20,7 +20,7 @@ func TestTaskCredentialsFlow(t *testing.T) {
}
resp := request("POST", "/api/tasks", `{
"description": "credential flow test",
"goal": "verify task credentials",
"goal": "Verify credential handling against the lab target at http://127.0.0.1:9/v1 and record the outcome.",
"credentials": [
{"domain": "target.example.com", "username": "alias@test", "password": "s3cret!", "note": "program alias account"},
{"domain": "target.example.com", "username": "", "password": ""}
@@ -32,7 +32,7 @@ func TestTaskCredentialsFlow(t *testing.T) {
resp = request("POST", "/api/tasks", `{
"description": "credential flow test",
"goal": "verify task credentials",
"goal": "Verify credential handling against the lab target at http://127.0.0.1:9/v1 and record the outcome.",
"credentials": [
{"domain": "target.example.com", "username": "alias@test", "password": "s3cret!", "note": "program alias account"},
{"domain": "app.target.example.com", "username": "agent-account", "password": ""}
+8
View File
@@ -120,6 +120,14 @@ func noteLLMCallFailure(taskID string, profileID int64) bool {
return false
}
// resetLLMFailStreaksForTest clears every streak; test isolation only (the
// escalation counter is package-global in-memory state).
func resetLLMFailStreaksForTest() {
llmFailStreaks.Lock()
llmFailStreaks.m = map[string]int{}
llmFailStreaks.Unlock()
}
func resetLLMFailStreak(taskID string, profileID int64) {
llmFailStreaks.Lock()
defer llmFailStreaks.Unlock()
+1 -6
View File
@@ -21,7 +21,7 @@ func (f *failingProvider) Complete(ctx context.Context, req llm.CompletionReques
}
func TestEscalationAdvancesAfterConsecutiveFailures(t *testing.T) {
resetAllLLMFailStreaksForTest()
resetLLMFailStreaksForTest()
prov := &failingProvider{err: errors.New("boom: upstream 500")}
profile := int64(7)
nextProfile := int64(8)
@@ -78,8 +78,3 @@ func TestEscalationAdvancesAfterConsecutiveFailures(t *testing.T) {
}
}
func resetAllLLMFailStreaksForTest() {
llmFailStreaks.Lock()
llmFailStreaks.m = map[string]int{}
llmFailStreaks.Unlock()
}
+1
View File
@@ -350,6 +350,7 @@ func TestTaskLLMStreamDoesNotSwitchForOrdinaryErrors(t *testing.T) {
}
for _, message := range tests {
t.Run(message, func(t *testing.T) {
resetLLMFailStreaksForTest() // package-global streaks must not leak across subtests
provider := &scriptedLLMProvider{err: errors.New(message)}
exhausted := 0
hooks := taskLLMStreamHooks{