mirror of
https://github.com/BaaRai/ARTEX.git
synced 2026-11-13 11:40:53 +08:00
fix(coverage): count auto-scope only under a declared root (F8)
Auto task_scope rows (source='auto') come from insert_assets and intent asset_ids, i.e. from target-derived content. They now count toward the coverage denominator and list_untested_assets only when they sit under a root the same task declared (a non-auto row: goal target registered via add_task_scope, or manual scope): the same or a deeper domain of a declared root_domain/subdomain, or a net inside a declared ip/cidr. The direct-source coverage view also stops counting an asset from an intent anchor alone; fact anchors still count, so tested evidence is unchanged. Without this, insert_assets put a planted host back into the backlog through its intent anchor. task_scope is described as "coverage scope, not enforced" in the code comments, the add_task_scope description, the goal-decomposer tail and the task overview UI, instead of "authorization boundary". Audit F8, 0.7.0.
This commit is contained in:
@@ -0,0 +1,106 @@
|
||||
package agent
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/Autumn-27/artex/db"
|
||||
)
|
||||
|
||||
// TestTaskScopeLabelIsHonest pins the task_scope wording while the RoE is not
|
||||
// back: the tool text and the goals tail call it coverage scope, not enforced, and
|
||||
// no longer claim an authorization boundary.
|
||||
func TestTaskScopeLabelIsHonest(t *testing.T) {
|
||||
texts := map[string]string{
|
||||
"add_task_scope": NewToolSet(nil, "").addTaskScope().Description(),
|
||||
"goalsScopeTail": goalsScopeTail,
|
||||
}
|
||||
for name, text := range texts {
|
||||
if !strings.Contains(text, taskScopeLabel) {
|
||||
t.Errorf("%s lacks %q", name, taskScopeLabel)
|
||||
}
|
||||
if strings.Contains(text, "authorization boundary") {
|
||||
t.Errorf("%s still claims an authorization boundary", name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestAutoScopeOutsideDeclaredRootExcluded: a worker inserts one host under the
|
||||
// declared root and one host outside it. Both get an auto scope row and an intent
|
||||
// anchor, but only the host under the declared root enters the coverage
|
||||
// denominator and list_untested_assets.
|
||||
func TestAutoScopeOutsideDeclaredRootExcluded(t *testing.T) {
|
||||
d := testDB(t)
|
||||
defer d.Close()
|
||||
|
||||
stamp := time.Now().UnixNano()
|
||||
declared := fmt.Sprintf("app.f8-%d.invalid", stamp)
|
||||
inside := "v1." + declared
|
||||
outsideRoot := fmt.Sprintf("f8-planted-%d.invalid", stamp)
|
||||
outside := "x." + outsideRoot
|
||||
|
||||
task, err := d.CreateTask("auto scope hygiene", "test "+declared, nil, 0, 0)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
assets := d.Assets()
|
||||
t.Cleanup(func() {
|
||||
_ = d.DeleteTask(task.ID)
|
||||
_, _ = d.Exec(`DELETE FROM assets WHERE domain IN ($1,$2,$3) OR root_domain IN ($4,$5)`,
|
||||
declared, inside, outside, fmt.Sprintf("f8-%d.invalid", stamp), outsideRoot)
|
||||
})
|
||||
// The goal target, as the goal decomposer registers it through add_task_scope.
|
||||
if _, err := assets.AddAgentScope(task.ID, "subdomain", declared, "goal target", "agent"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
store := d.Exploration(task.ExplorationID)
|
||||
intentID, err := store.AddNode(db.KindIntent, map[string]any{"summary": "recon"}, 0, "open", "planner", nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
worker := NewToolSet(store, "worker")
|
||||
worker.SetTaskID(task.ID)
|
||||
worker.SetAssetStore(assets, d.Companies())
|
||||
worker.SetOwnerNode(intentID)
|
||||
out := callInsertAssets(t, worker, map[string]any{"assets": []any{
|
||||
map[string]any{"type": "subdomain", "domain": inside},
|
||||
map[string]any{"type": "subdomain", "domain": outside},
|
||||
}})
|
||||
if errs, _ := out["errors"].([]any); len(errs) > 0 {
|
||||
t.Fatalf("insert_assets errors: %v", errs)
|
||||
}
|
||||
scope, err := assets.ListTaskScope(task.ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
autoRows := map[string]bool{}
|
||||
for _, row := range scope {
|
||||
if row.Source == "auto" {
|
||||
autoRows[row.Domain] = true
|
||||
}
|
||||
}
|
||||
if !autoRows[inside] || !autoRows[outside] {
|
||||
t.Fatalf("both hosts should be auto-scoped: %+v", scope)
|
||||
}
|
||||
|
||||
planner := NewToolSet(store, "planner")
|
||||
planner.SetTaskID(task.ID)
|
||||
planner.SetAssetStore(assets, d.Companies())
|
||||
coverage, ok := planner.graphOverviewData()["coverage"].(map[string]any)
|
||||
if !ok {
|
||||
t.Fatal("coverage block missing from graph overview")
|
||||
}
|
||||
if got := coverage["denominator"]; got != 1 {
|
||||
t.Fatalf("coverage denominator = %v, want 1 (only %s)", got, inside)
|
||||
}
|
||||
untested := fmt.Sprint(callReadJSON(t, planner.listUntestedAssets(), `{"page":1,"page_size":50}`))
|
||||
if !strings.Contains(untested, inside) {
|
||||
t.Fatalf("host under the declared root missing from list_untested_assets: %s", untested)
|
||||
}
|
||||
if strings.Contains(untested, outsideRoot) {
|
||||
t.Fatalf("host outside the declared root listed as untested: %s", untested)
|
||||
}
|
||||
}
|
||||
+1
-1
@@ -53,7 +53,7 @@ Call set_goals to submit the results.`
|
||||
const goalsScopeTail = `
|
||||
|
||||
**Extra duty: register the test asset scope**
|
||||
Besides decomposing goals, you must also identify the **explicitly stated test asset scope** from the "task goal / task description" and register it via add_task_scope (this task's authorization boundary, and also the denominator of asset test coverage). **Minimal-scope principle: register only the one target the user explicitly named — never enlarge it on your own.**
|
||||
Besides decomposing goals, you must also identify the **explicitly stated test asset scope** from the "task goal / task description" and register it via add_task_scope (this task's ` + taskScopeLabel + `: the denominator of asset test coverage, and the root under which auto-added hosts count). **Minimal-scope principle: register only the one target the user explicitly named — never enlarge it on your own.**
|
||||
- The goal is a URL or a host-bearing address (e.g. https://xxx.example.com/path, app.example.com) → take its **full hostname**, kind=subdomain, value=full hostname.
|
||||
Example: goal https://a1b2c3.lab.example.net/path → kind=subdomain, value=a1b2c3.lab.example.net (**not** example.net).
|
||||
**Never** shrink a subdomain-bearing hostname to its root domain — registering all of example.com upon seeing xxx.example.com expands the scope beyond the user's target, violating the minimal-scope principle.
|
||||
|
||||
+12
-3
@@ -364,6 +364,11 @@ func (t *ToolSet) insertAssets() actool.CoreTool {
|
||||
// inserted by the worker at top level, adding a conservative scope per its
|
||||
// type; side-effect-derived assets don't go through here, so the scope
|
||||
// doesn't widen blindly. No-op when taskID=0.
|
||||
// The item may come from target content, so this auto row counts toward
|
||||
// coverage and list_untested_assets only under a declared root (a
|
||||
// non-auto row: goal target, add_task_scope or manual scope, see
|
||||
// db.declaredScopeFilter). task_scope is
|
||||
// coverage scope, not enforced: nothing blocks a call outside it.
|
||||
// Independent of the coverage switch: task_scope is the task's range
|
||||
// boundary (the filter basis for list/query); the coverage switch only
|
||||
// decides whether to use it as the denominator for metrics, not whether
|
||||
@@ -436,16 +441,20 @@ func (t *ToolSet) addCompanyScope() actool.CoreTool {
|
||||
)
|
||||
}
|
||||
|
||||
// taskScopeLabel is the honest description of task_scope while the RoE is not back:
|
||||
// it drives coverage and asset filters, and nothing enforces it on tool calls.
|
||||
const taskScopeLabel = "coverage scope, not enforced"
|
||||
|
||||
// addTaskScope lets the plan agent add test scope to THE CURRENT TASK — the coverage
|
||||
// denominator and the task's authorization edge. Worker discoveries are auto-scoped
|
||||
// denominator (coverage scope, not enforced). Worker discoveries are auto-scoped
|
||||
// (precise host) by insertAssets; this tool is for DELIBERATELY WIDENING: pull a whole
|
||||
// root domain or whole company into scope, or add a specific subdomain / ip.
|
||||
func (t *ToolSet) addTaskScope() actool.CoreTool {
|
||||
return writeTool(
|
||||
"add_task_scope",
|
||||
"Add test scope to [this task] — this is the task's authorization boundary, and also the denominator of asset test coverage.\n"+
|
||||
"Add test scope to [this task] — "+taskScopeLabel+": it is the denominator of asset test coverage; nothing blocks a tool call outside it.\n"+
|
||||
"Supported kinds: company (all assets under the company) / root_domain (a whole root domain, including all subdomains) / subdomain (a single precise subdomain) / ip / cidr / icp / keyword.\n"+
|
||||
"Note: hosts a worker touches one by one are [automatically] added to the scope by the system (precise subdomains); this tool is for [deliberate widening] — pulling in a whole root domain / whole company, or adding a specific subdomain/IP.\n"+
|
||||
"Note: hosts a worker touches one by one are [automatically] added to the scope by the system (precise subdomains), but they count toward coverage and list_untested_assets only under a declared root (a goal target or scope added here or manually); this tool is for [deliberate widening] — pulling in a whole root domain / whole company, or adding a specific subdomain/IP.\n"+
|
||||
"value: for company pass the company name or id (the company must already exist); for root_domain/subdomain pass a domain; for ip/cidr pass an IP or range; for icp/keyword pass an ICP number or company keyword.\n"+
|
||||
"Always give reason explaining the basis (auditable). Use the entries array for multiple entries.",
|
||||
obj(map[string]any{
|
||||
|
||||
@@ -454,6 +454,9 @@ func (w *Worker) execute(ctx context.Context, name string, taskID int64, as *db.
|
||||
// test scope (same conservative granularity as insertAssets). upsertTaskScope's
|
||||
// ON CONFLICT DO NOTHING + the uq_task_scope unique index guarantee no
|
||||
// duplicate additions; reruns/retries are idempotent no-ops as well.
|
||||
// asset_ids can trace back to target content, so these auto rows count
|
||||
// toward coverage only under a declared root (db.declaredScopeFilter);
|
||||
// coverage scope, not enforced.
|
||||
// When the asset-coverage feature is off, the test scope (denominator) is no
|
||||
// longer accumulated.
|
||||
if coverageEnabled {
|
||||
|
||||
+2
-1
@@ -754,12 +754,13 @@ WHERE t.active_llm_profile_id IS NULL
|
||||
AND t.llm_profile_id IS NOT NULL
|
||||
AND EXISTS (SELECT 1 FROM task_llm_profiles x WHERE x.task_id=t.id AND x.profile_id=t.llm_profile_id);
|
||||
|
||||
-- Task test scope (the asset-coverage denominator + authorization boundary).
|
||||
-- Task test scope (the asset-coverage denominator; coverage scope, not enforced).
|
||||
-- Auto-filled (source='auto'): insertAssets' top level adds a conservative scope per asset type explicitly
|
||||
-- inserted by the worker (root_domain→root_domain, subdomain/service/endpoint→subdomain(host), ip→ip);
|
||||
-- side-effect-derived assets do not enter the scope (the hook sits at the handler top level; derivation is internal to the db layer).
|
||||
-- Agent-filled (source='agent'): add_task_scope adds company/root_domain/subdomain/ip.
|
||||
-- Coverage = among assets matching active rows (denominator), the share anchored by fact nodes (numerator).
|
||||
-- An auto row is active only under a declared (non-auto) root of the same task (declaredScopeFilter in task_scope.go).
|
||||
CREATE TABLE IF NOT EXISTS task_scope (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
task_id BIGINT NOT NULL REFERENCES tasks(id) ON DELETE CASCADE,
|
||||
|
||||
@@ -37,12 +37,13 @@ target AS (
|
||||
))
|
||||
)
|
||||
JOIN context_tasks ctx ON ctx.task_id=ts.task_id
|
||||
WHERE ` + declaredScopeFilter + `
|
||||
UNION
|
||||
SELECT a.id, a.type,
|
||||
COALESCE(a.url, a.domain, a.ip, a.app_name, a.root_domain, '') AS label
|
||||
FROM assets a
|
||||
JOIN exploration_anchors ea ON ea.asset_id=a.id
|
||||
JOIN exploration_nodes en ON en.id=ea.node_id
|
||||
JOIN exploration_nodes en ON en.id=ea.node_id AND en.kind='fact'
|
||||
JOIN context_tasks ctx ON ctx.exploration_id=en.exploration_id
|
||||
),
|
||||
tested AS (
|
||||
@@ -118,8 +119,10 @@ ORDER BY CASE WHEN ts.task_id=$1 THEN 0 ELSE 1 END, ts.id`, taskID)
|
||||
}
|
||||
|
||||
// TaskCoverageWithSources computes one coverage view over the union of the
|
||||
// current task and its direct sources: source scopes and anchored assets extend
|
||||
// the denominator, while fact anchors count as tested. No row is copied.
|
||||
// current task and its direct sources: source scopes and fact-anchored assets
|
||||
// extend the denominator, while fact anchors count as tested. Auto scope rows
|
||||
// count only under a declared root (declaredScopeFilter), and an intent anchor
|
||||
// alone (insert_assets, asset_ids) no longer adds an asset. No row is copied.
|
||||
func (s *AssetStore) TaskCoverageWithSources(taskID int64) (*Coverage, error) {
|
||||
cov := &Coverage{ByType: []CoverageByType{}}
|
||||
_ = s.db.QueryRow(`WITH `+directTaskContextCTE+`
|
||||
|
||||
+29
-7
@@ -8,10 +8,13 @@ import (
|
||||
"strings"
|
||||
)
|
||||
|
||||
// TaskScope is one row of a task's test scope — the coverage denominator and the
|
||||
// per-task authorization edge. Rows come either from insertAssets (source='auto',
|
||||
// conservative, one per explicitly-inserted asset) or from the add_task_scope tool
|
||||
// (source='agent', for company / domain / network / ICP / keyword scope).
|
||||
// TaskScope is one row of a task's test scope — coverage scope, not enforced: it is
|
||||
// the coverage denominator and the filter for asset queries, and nothing blocks a
|
||||
// tool call outside it. Rows come from insertAssets / intent asset_ids (source='auto',
|
||||
// conservative, one per explicitly-inserted asset), from the add_task_scope tool
|
||||
// (source='agent': the goal targets the decomposer registers, or planner widening),
|
||||
// or from the operator (source='manual'). An auto row counts toward coverage only
|
||||
// under a declared (non-auto) root, see declaredScopeFilter.
|
||||
type TaskScope struct {
|
||||
ID int64 `json:"id"`
|
||||
TaskID int64 `json:"task_id"`
|
||||
@@ -105,6 +108,8 @@ func (s *AssetStore) upsertTaskScope(ts TaskScope) error {
|
||||
// asset item (source='auto'). MUST be called only from insertAssets' top-level loop —
|
||||
// never from a db-layer side effect (linkHostAssets), so derived assets do not blindly expand the scope.
|
||||
// Rule: scope granularity follows the asset's own type. taskID<=0 → no-op.
|
||||
// The asset comes from target-derived content, so the row counts toward coverage and
|
||||
// list_untested_assets only under a declared root (declaredScopeFilter).
|
||||
func (s *AssetStore) AddAutoScope(taskID int64, assetType, domain, rawURL, ip string) error {
|
||||
if taskID <= 0 {
|
||||
return nil
|
||||
@@ -274,8 +279,9 @@ type CoverageByType struct {
|
||||
}
|
||||
|
||||
// Coverage is a task's rough asset test coverage — a reference figure for the agent,
|
||||
// NOT a precise metric. Denominator = assets matching any active task_scope row;
|
||||
// Tested = those anchored to at least one fact node in the exploration.
|
||||
// NOT a precise metric. Denominator = assets matching a declared task_scope row or an
|
||||
// auto row under a declared root (declaredScopeFilter); Tested = those anchored to at
|
||||
// least one fact node in the exploration.
|
||||
type Coverage struct {
|
||||
Enabled bool `json:"enabled"` // whether the asset coverage feature is on; when false the other fields are zero
|
||||
ScopeRows int `json:"scope_rows"` // 0 → scope not anchored
|
||||
@@ -299,6 +305,22 @@ func (s *AssetStore) CoverageEnabled(taskID int64) bool {
|
||||
return enabled
|
||||
}
|
||||
|
||||
// declaredScopeFilter keeps a task_scope row ts in the coverage match only when it is
|
||||
// declared (source<>'auto': goal targets registered through add_task_scope, or manual
|
||||
// scope) or when it is an auto row under a root the same task declared: the same or
|
||||
// a deeper domain of a declared root_domain/subdomain, or a net inside a declared
|
||||
// ip/cidr. Auto rows come from target-derived content (insert_assets, intent
|
||||
// asset_ids), so anything else stays out of the coverage denominator and
|
||||
// list_untested_assets.
|
||||
const declaredScopeFilter = `(ts.source <> 'auto' OR EXISTS (
|
||||
SELECT 1 FROM task_scope d
|
||||
WHERE d.task_id = ts.task_id AND d.source <> 'auto' AND (
|
||||
(d.kind IN ('root_domain','subdomain') AND ts.domain IS NOT NULL
|
||||
AND (ts.domain = d.domain OR right(ts.domain, length(d.domain)+1) = '.' || d.domain))
|
||||
OR (d.kind IN ('ip','cidr') AND ts.net IS NOT NULL AND ts.net <<= d.net)
|
||||
)
|
||||
))`
|
||||
|
||||
// task_scope→assets match predicate, reused by the count / by-type / untested queries. $1=taskID.
|
||||
const covTargetCTE = `
|
||||
target AS (
|
||||
@@ -314,7 +336,7 @@ target AS (
|
||||
lower(regexp_replace(COALESCE(a.icp,''), '[[:space:]]+', '', 'g')) = ts.value
|
||||
OR lower(regexp_replace(COALESCE(a.app_icp,''), '[[:space:]]+', '', 'g')) = ts.value
|
||||
))
|
||||
)
|
||||
) AND ` + declaredScopeFilter + `
|
||||
),
|
||||
tested AS (
|
||||
SELECT DISTINCT ea.asset_id
|
||||
|
||||
@@ -805,13 +805,13 @@ export function OverviewTab({ taskId }: { taskId: string }) {
|
||||
)}
|
||||
</CardContent>
|
||||
</Card>
|
||||
{/* Test scope: coverage denominator + authorization boundary; can be edited manually. */}
|
||||
{/* Test scope: coverage denominator (coverage scope, not enforced); can be edited manually. */}
|
||||
<Card>
|
||||
<CardHeader>
|
||||
<CardTitle className="flex items-center gap-2 text-base">
|
||||
<ShieldCheckIcon className="size-4 text-emerald-500" /> Test scope
|
||||
<span className="text-muted-foreground text-xs font-normal">
|
||||
(coverage denominator + authorization boundary, {scope.length} in total)
|
||||
(coverage scope, not enforced, {scope.length} in total)
|
||||
</span>
|
||||
</CardTitle>
|
||||
</CardHeader>
|
||||
|
||||
@@ -350,7 +350,7 @@ export interface WorkspaceFile {
|
||||
content?: string;
|
||||
}
|
||||
|
||||
// One row of a task's test scope (coverage denominator + authorization boundary).
|
||||
// One row of a task's test scope (coverage denominator; coverage scope, not enforced).
|
||||
export interface TaskScopeRow {
|
||||
id: number;
|
||||
task_id: number;
|
||||
|
||||
Reference in New Issue
Block a user