pphua
9793b522d9
Add read-only task context inheritance, ordered task-level LLM failover, live Worker controls, cascading task cleanup, FIFO task concurrency, cancellation diagnostics, UI updates, tests, build script, and changelog.
678 lines
35 KiB
PL/PgSQL
678 lines
35 KiB
PL/PgSQL
-- ARTEX PostgreSQL schema (单一数据源)
|
||
-- 幂等：可重复执行（IF NOT EXISTS / OR REPLACE / DROP TRIGGER IF EXISTS）。
|
||
|
||
-- =====================================================================
|
||
-- 0. 通用：updated_at 触发器
|
||
-- =====================================================================
|
||
CREATE OR REPLACE FUNCTION set_updated_at() RETURNS trigger AS $$
|
||
BEGIN NEW.updated_at = now(); RETURN NEW; END;
|
||
$$ LANGUAGE plpgsql;
|
||
|
||
-- =====================================================================
|
||
-- A. 资产层：companies / assets / company_scope
|
||
-- =====================================================================
|
||
|
||
CREATE TABLE IF NOT EXISTS companies (
|
||
id BIGSERIAL PRIMARY KEY,
|
||
name TEXT NOT NULL,
|
||
nkey TEXT NOT NULL UNIQUE,
|
||
logo TEXT,
|
||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||
updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||
);
|
||
CREATE INDEX IF NOT EXISTS idx_companies_nkey ON companies(nkey);
|
||
DROP TRIGGER IF EXISTS trg_companies_upd ON companies;
|
||
CREATE TRIGGER trg_companies_upd BEFORE UPDATE ON companies
|
||
FOR EACH ROW EXECUTE FUNCTION set_updated_at();
|
||
|
||
CREATE TABLE IF NOT EXISTS assets (
|
||
id BIGSERIAL PRIMARY KEY,
|
||
type TEXT NOT NULL CHECK (type IN (
|
||
'root_domain','ip','subdomain','app','service','endpoint'
|
||
)),
|
||
company_id BIGINT REFERENCES companies(id) ON DELETE SET NULL,
|
||
task_ids BIGINT[] NOT NULL DEFAULT '{}',
|
||
domain TEXT,
|
||
root_domain TEXT,
|
||
ip TEXT,
|
||
c_segment CIDR,
|
||
port INTEGER CHECK (port BETWEEN 1 AND 65535),
|
||
icp TEXT,
|
||
bound_domains TEXT[] NOT NULL DEFAULT '{}',
|
||
open_ports JSONB[] NOT NULL DEFAULT '{}',
|
||
record_type TEXT,
|
||
record_value TEXT[],
|
||
bundle_id TEXT,
|
||
app_name TEXT,
|
||
category TEXT,
|
||
app_description TEXT,
|
||
app_icp TEXT,
|
||
url TEXT,
|
||
service_type TEXT CHECK (service_type IN ('http','other')),
|
||
service_name TEXT,
|
||
favicon_mmh3 TEXT,
|
||
status_code INTEGER,
|
||
content_length BIGINT,
|
||
page_title TEXT,
|
||
technologies TEXT[] NOT NULL DEFAULT '{}',
|
||
auth JSONB[] NOT NULL DEFAULT '{}',
|
||
method TEXT,
|
||
params JSONB[] NOT NULL DEFAULT '{}',
|
||
extra JSONB NOT NULL DEFAULT '{}',
|
||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||
last_seen TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||
updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||
);
|
||
CREATE UNIQUE INDEX IF NOT EXISTS uq_av2_root_domain ON assets(domain) WHERE type = 'root_domain';
|
||
CREATE UNIQUE INDEX IF NOT EXISTS uq_av2_ip ON assets(ip) WHERE type = 'ip';
|
||
CREATE UNIQUE INDEX IF NOT EXISTS uq_av2_subdomain ON assets(domain, COALESCE(record_type,'')) WHERE type = 'subdomain';
|
||
CREATE UNIQUE INDEX IF NOT EXISTS uq_av2_app_bundle ON assets(bundle_id) WHERE type = 'app' AND bundle_id IS NOT NULL;
|
||
CREATE UNIQUE INDEX IF NOT EXISTS uq_av2_app_name ON assets(app_name) WHERE type = 'app' AND bundle_id IS NULL;
|
||
CREATE UNIQUE INDEX IF NOT EXISTS uq_av2_service_http ON assets(url) WHERE type = 'service' AND service_type = 'http';
|
||
CREATE UNIQUE INDEX IF NOT EXISTS uq_av2_service_other
|
||
ON assets(COALESCE(domain,''), COALESCE(ip,''), port, service_name) WHERE type = 'service' AND service_type = 'other';
|
||
CREATE UNIQUE INDEX IF NOT EXISTS uq_av2_endpoint ON assets(url, method) WHERE type = 'endpoint';
|
||
CREATE INDEX IF NOT EXISTS idx_av2_company ON assets(company_id) WHERE company_id IS NOT NULL;
|
||
CREATE INDEX IF NOT EXISTS idx_av2_company_type ON assets(company_id, type) WHERE company_id IS NOT NULL;
|
||
CREATE INDEX IF NOT EXISTS idx_av2_task_ids ON assets USING GIN(task_ids);
|
||
CREATE INDEX IF NOT EXISTS idx_av2_domain ON assets(domain) WHERE domain IS NOT NULL;
|
||
CREATE INDEX IF NOT EXISTS idx_av2_root_domain ON assets(root_domain) WHERE root_domain IS NOT NULL;
|
||
CREATE INDEX IF NOT EXISTS idx_av2_ip ON assets(ip) WHERE ip IS NOT NULL;
|
||
CREATE INDEX IF NOT EXISTS idx_av2_c_segment ON assets USING GIST(c_segment inet_ops) WHERE c_segment IS NOT NULL;
|
||
CREATE INDEX IF NOT EXISTS idx_av2_technologies ON assets USING GIN(technologies) WHERE type = 'service';
|
||
CREATE INDEX IF NOT EXISTS idx_av2_bound_domains ON assets USING GIN(bound_domains) WHERE type = 'ip';
|
||
CREATE INDEX IF NOT EXISTS idx_av2_open_ports ON assets USING GIN(open_ports) WHERE type = 'ip';
|
||
CREATE INDEX IF NOT EXISTS idx_av2_last_seen ON assets(last_seen DESC);
|
||
CREATE INDEX IF NOT EXISTS idx_av2_type_seen ON assets(type, last_seen DESC);
|
||
DROP TRIGGER IF EXISTS trg_av2_upd ON assets;
|
||
CREATE TRIGGER trg_av2_upd BEFORE UPDATE ON assets
|
||
FOR EACH ROW EXECUTE FUNCTION set_updated_at();
|
||
|
||
CREATE TABLE IF NOT EXISTS company_scope (
|
||
id BIGSERIAL PRIMARY KEY,
|
||
company_id BIGINT NOT NULL REFERENCES companies(id) ON DELETE CASCADE,
|
||
kind TEXT NOT NULL CHECK (kind IN ('domain','ip','cidr')),
|
||
domain TEXT,
|
||
net CIDR,
|
||
raw TEXT NOT NULL,
|
||
reason TEXT,
|
||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||
CONSTRAINT uq_sv2_domain UNIQUE (company_id, domain),
|
||
CONSTRAINT uq_sv2_net UNIQUE (company_id, net)
|
||
);
|
||
CREATE INDEX IF NOT EXISTS idx_sv2_domain ON company_scope(domain) WHERE kind = 'domain';
|
||
CREATE INDEX IF NOT EXISTS idx_sv2_net ON company_scope USING GIST(net inet_ops) WHERE kind IN ('ip','cidr');
|
||
CREATE INDEX IF NOT EXISTS idx_sv2_company ON company_scope(company_id);
|
||
|
||
-- =====================================================================
|
||
-- B. 推理探索层
|
||
-- =====================================================================
|
||
CREATE TABLE IF NOT EXISTS explorations (
|
||
id BIGSERIAL PRIMARY KEY,
|
||
description TEXT,
|
||
goal TEXT NOT NULL,
|
||
status TEXT NOT NULL DEFAULT 'open'
|
||
CHECK (status IN ('open','achieved','failed')),
|
||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||
updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||
);
|
||
DROP TRIGGER IF EXISTS trg_exp_upd ON explorations;
|
||
CREATE TRIGGER trg_exp_upd BEFORE UPDATE ON explorations
|
||
FOR EACH ROW EXECUTE FUNCTION set_updated_at();
|
||
|
||
CREATE TABLE IF NOT EXISTS exploration_nodes (
|
||
id BIGSERIAL PRIMARY KEY,
|
||
exploration_id BIGINT NOT NULL REFERENCES explorations(id) ON DELETE CASCADE,
|
||
kind TEXT NOT NULL,
|
||
payload JSONB NOT NULL DEFAULT '{}',
|
||
priority INT NOT NULL DEFAULT 0,
|
||
state TEXT NOT NULL DEFAULT 'open',
|
||
origin TEXT,
|
||
owner TEXT,
|
||
blocked_reason TEXT,
|
||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||
updated_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||
completed_at TIMESTAMPTZ,
|
||
CONSTRAINT ck_node_kind CHECK (kind IN ('begin','goal','intent','fact','finding','hint')),
|
||
CONSTRAINT ck_node_state CHECK (
|
||
(kind='begin' AND state IN ('open')) OR
|
||
(kind='intent' AND state IN ('open','running','paused','done','blocked','exhausted','stopped')) OR
|
||
(kind='goal' AND state IN ('open','met','abandoned')) OR
|
||
(kind='fact' AND state IN ('confirmed','dismissed','origin')) OR
|
||
(kind='finding' AND state IN ('confirmed','dismissed')) OR
|
||
(kind='hint' AND state IN ('active','consumed'))
|
||
)
|
||
);
|
||
ALTER TABLE exploration_nodes ADD COLUMN IF NOT EXISTS blocked_reason TEXT;
|
||
DO $$
|
||
BEGIN
|
||
IF EXISTS (
|
||
SELECT 1 FROM pg_constraint
|
||
WHERE conrelid='exploration_nodes'::regclass
|
||
AND conname='ck_node_state'
|
||
AND pg_get_constraintdef(oid) NOT LIKE '%paused%'
|
||
) THEN
|
||
ALTER TABLE exploration_nodes DROP CONSTRAINT ck_node_state;
|
||
ALTER TABLE exploration_nodes ADD CONSTRAINT ck_node_state CHECK (
|
||
(kind='begin' AND state IN ('open')) OR
|
||
(kind='intent' AND state IN ('open','running','paused','done','blocked','exhausted','stopped')) OR
|
||
(kind='goal' AND state IN ('open','met','abandoned')) OR
|
||
(kind='fact' AND state IN ('confirmed','dismissed','origin')) OR
|
||
(kind='finding' AND state IN ('confirmed','dismissed')) OR
|
||
(kind='hint' AND state IN ('active','consumed'))
|
||
);
|
||
END IF;
|
||
END $$;
|
||
CREATE INDEX IF NOT EXISTS idx_expnodes_part ON exploration_nodes(exploration_id, kind);
|
||
CREATE INDEX IF NOT EXISTS idx_expnodes_frontier ON exploration_nodes(exploration_id, priority DESC)
|
||
WHERE kind='intent' AND state='open';
|
||
DROP TRIGGER IF EXISTS trg_expnodes_upd ON exploration_nodes;
|
||
CREATE TRIGGER trg_expnodes_upd BEFORE UPDATE ON exploration_nodes
|
||
FOR EACH ROW EXECUTE FUNCTION set_updated_at();
|
||
|
||
CREATE TABLE IF NOT EXISTS exploration_edges (
|
||
exploration_id BIGINT NOT NULL REFERENCES explorations(id) ON DELETE CASCADE,
|
||
src_id BIGINT NOT NULL REFERENCES exploration_nodes(id) ON DELETE CASCADE,
|
||
dst_id BIGINT NOT NULL REFERENCES exploration_nodes(id) ON DELETE CASCADE,
|
||
rel TEXT NOT NULL CHECK (rel IN ('spawns','derived_from','yields','proves')),
|
||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||
PRIMARY KEY (exploration_id, src_id, rel, dst_id),
|
||
CONSTRAINT ck_edge_noself CHECK (src_id <> dst_id)
|
||
);
|
||
CREATE INDEX IF NOT EXISTS idx_expedges_src ON exploration_edges(src_id, rel);
|
||
CREATE INDEX IF NOT EXISTS idx_expedges_dst ON exploration_edges(dst_id, rel);
|
||
|
||
CREATE TABLE IF NOT EXISTS exploration_anchors (
|
||
node_id BIGINT NOT NULL REFERENCES exploration_nodes(id) ON DELETE CASCADE,
|
||
asset_id BIGINT NOT NULL REFERENCES assets(id) ON DELETE CASCADE,
|
||
PRIMARY KEY (node_id, asset_id)
|
||
);
|
||
CREATE INDEX IF NOT EXISTS idx_anchor_asset ON exploration_anchors(asset_id);
|
||
|
||
CREATE TABLE IF NOT EXISTS activity (
|
||
id BIGSERIAL PRIMARY KEY,
|
||
exploration_id BIGINT NOT NULL REFERENCES explorations(id) ON DELETE CASCADE,
|
||
node_id BIGINT REFERENCES exploration_nodes(id) ON DELETE SET NULL,
|
||
worker TEXT,
|
||
kind TEXT,
|
||
tool TEXT,
|
||
tool_use_id TEXT,
|
||
is_error BOOLEAN NOT NULL DEFAULT false,
|
||
summary TEXT,
|
||
detail TEXT,
|
||
metadata JSONB NOT NULL DEFAULT '{}',
|
||
input_tokens INTEGER,
|
||
output_tokens INTEGER,
|
||
cache_read_tokens INTEGER,
|
||
cache_write_tokens INTEGER,
|
||
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||
);
|
||
ALTER TABLE activity ADD COLUMN IF NOT EXISTS metadata JSONB NOT NULL DEFAULT '{}';
|
||
CREATE INDEX IF NOT EXISTS idx_act_node ON activity(exploration_id, node_id, id);
|
||
CREATE INDEX IF NOT EXISTS idx_act_since ON activity(exploration_id, id);
|
||
-- Main/Plan history pages filter by worker (both carry NULL node_id, so idx_act_node
|
||
-- can't distinguish them); this covers reverse pagination of those sessions.
|
||
CREATE INDEX IF NOT EXISTS idx_act_worker ON activity(exploration_id, worker, id);
|
||
|
||
-- =====================================================================
|
||
-- C. LLM profiles
|
||
-- =====================================================================
|
||
CREATE TABLE IF NOT EXISTS settings (
|
||
key TEXT PRIMARY KEY,
|
||
value TEXT NOT NULL,
|
||
updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||
);
|
||
|
||
CREATE TABLE IF NOT EXISTS llm_profiles (
|
||
id BIGSERIAL PRIMARY KEY,
|
||
name TEXT NOT NULL UNIQUE,
|
||
format TEXT NOT NULL CHECK (format IN ('openai','anthropic')),
|
||
base_url TEXT,
|
||
proxy TEXT,
|
||
model TEXT NOT NULL,
|
||
api_key TEXT,
|
||
api_key_hint TEXT,
|
||
rate_per_second DOUBLE PRECISION NOT NULL DEFAULT 0,
|
||
rate_per_minute DOUBLE PRECISION NOT NULL DEFAULT 0,
|
||
context_window_k INTEGER NOT NULL DEFAULT 0,
|
||
reasoning_effort TEXT NOT NULL DEFAULT '',
|
||
is_default BOOLEAN NOT NULL DEFAULT false,
|
||
-- 轮询(故障转移)参数，见 docs/LLM轮询设计.md：
|
||
-- priority 顺位，越大越先被选中；激活配置(is_default)永远排链首，与本值无关。
|
||
-- pool_exclude true=不作为故障转移目标(仍可被 agent/任务显式绑定使用)。
|
||
priority INTEGER NOT NULL DEFAULT 0,
|
||
pool_exclude BOOLEAN NOT NULL DEFAULT false,
|
||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||
updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||
);
|
||
CREATE UNIQUE INDEX IF NOT EXISTS uq_llm_one_default ON llm_profiles(is_default) WHERE is_default;
|
||
DROP TRIGGER IF EXISTS trg_llm_upd ON llm_profiles;
|
||
CREATE TRIGGER trg_llm_upd BEFORE UPDATE ON llm_profiles
|
||
FOR EACH ROW EXECUTE FUNCTION set_updated_at();
|
||
-- 轮询顺位/排除标记；补旧库。默认 0 / false = 全部配置都参与轮询。
|
||
ALTER TABLE llm_profiles ADD COLUMN IF NOT EXISTS priority INTEGER NOT NULL DEFAULT 0;
|
||
ALTER TABLE llm_profiles ADD COLUMN IF NOT EXISTS pool_exclude BOOLEAN NOT NULL DEFAULT false;
|
||
|
||
-- LLM 轮询熔断状态：某个配置连续失败(余额不足/key 失效/限流)后进入冷却，冷却期内
|
||
-- 轮询直接跳过它。内存态为准，这里落库只为重启后不丢冷却窗口——加载时只取尚未
|
||
-- 到期的行(open_until > now)，已到期的自然回到"正常"，等下一次调用半开试探。
|
||
CREATE TABLE IF NOT EXISTS llm_profile_health (
|
||
profile_id BIGINT PRIMARY KEY REFERENCES llm_profiles(id) ON DELETE CASCADE,
|
||
fails INTEGER NOT NULL DEFAULT 0, -- 当前连续失败次数(成功即清零)
|
||
trips INTEGER NOT NULL DEFAULT 0, -- 累计熔断次数,用于冷却时间指数退避
|
||
open_until TIMESTAMPTZ, -- 冷却截止;NULL/过期 = 未熔断
|
||
last_error TEXT NOT NULL DEFAULT '',
|
||
last_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||
);
|
||
|
||
-- =====================================================================
|
||
-- D. 任务层
|
||
-- =====================================================================
|
||
CREATE TABLE IF NOT EXISTS tasks (
|
||
id BIGSERIAL PRIMARY KEY,
|
||
description TEXT NOT NULL,
|
||
goal TEXT NOT NULL,
|
||
exploration_id BIGINT NOT NULL UNIQUE
|
||
REFERENCES explorations(id) ON DELETE RESTRICT,
|
||
status TEXT NOT NULL DEFAULT 'created'
|
||
CHECK (status IN ('created','running','paused','done','failed','timeout')),
|
||
paused BOOLEAN NOT NULL DEFAULT false,
|
||
queued BOOLEAN NOT NULL DEFAULT false,
|
||
llm_profile_id BIGINT REFERENCES llm_profiles(id) ON DELETE SET NULL,
|
||
active_llm_profile_id BIGINT REFERENCES llm_profiles(id) ON DELETE SET NULL,
|
||
llm_chain_revision BIGINT NOT NULL DEFAULT 0,
|
||
company_id BIGINT REFERENCES companies(id) ON DELETE SET NULL,
|
||
parent_ref TEXT,
|
||
timeout_seconds INTEGER NOT NULL DEFAULT 0,
|
||
plan_heartbeat_seconds INTEGER NOT NULL DEFAULT 300,
|
||
first_run_at TIMESTAMPTZ,
|
||
deadline_at TIMESTAMPTZ,
|
||
deleted_at TIMESTAMPTZ,
|
||
completed_at TIMESTAMPTZ,
|
||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||
updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||
);
|
||
CREATE INDEX IF NOT EXISTS idx_tasks_alive ON tasks(created_at DESC) WHERE deleted_at IS NULL;
|
||
CREATE INDEX IF NOT EXISTS idx_tasks_status ON tasks(status) WHERE deleted_at IS NULL;
|
||
DROP TRIGGER IF EXISTS trg_tasks_upd ON tasks;
|
||
CREATE TRIGGER trg_tasks_upd BEFORE UPDATE ON tasks
|
||
FOR EACH ROW EXECUTE FUNCTION set_updated_at();
|
||
-- planner 心跳触发间隔(秒);补旧库。默认 300s(5min)。见 docs/planner-trigger-impl-plan.md
|
||
ALTER TABLE tasks ADD COLUMN IF NOT EXISTS plan_heartbeat_seconds INTEGER NOT NULL DEFAULT 300;
|
||
-- 并发上限挂起态;补旧库。true=因并发上限排队、等待空位自动启动。
|
||
ALTER TABLE tasks ADD COLUMN IF NOT EXISTS queued BOOLEAN NOT NULL DEFAULT false;
|
||
ALTER TABLE tasks ADD COLUMN IF NOT EXISTS active_llm_profile_id BIGINT REFERENCES llm_profiles(id) ON DELETE SET NULL;
|
||
ALTER TABLE tasks ADD COLUMN IF NOT EXISTS llm_chain_revision BIGINT NOT NULL DEFAULT 0;
|
||
|
||
-- Direct, read-only task context inheritance. Relations are intentionally not
|
||
-- recursive: a task sees only the source tasks explicitly chosen at creation.
|
||
CREATE TABLE IF NOT EXISTS task_relations (
|
||
task_id BIGINT NOT NULL REFERENCES tasks(id) ON DELETE CASCADE,
|
||
source_task_id BIGINT NOT NULL REFERENCES tasks(id) ON DELETE CASCADE,
|
||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||
PRIMARY KEY (task_id, source_task_id),
|
||
CONSTRAINT ck_task_relation_not_self CHECK (task_id <> source_task_id)
|
||
);
|
||
CREATE INDEX IF NOT EXISTS idx_task_relations_source ON task_relations(source_task_id);
|
||
|
||
-- Ordered task-level LLM failover chain. A quota-exhausted entry is skipped
|
||
-- until the user saves/resets the chain, which clears all failure state.
|
||
CREATE TABLE IF NOT EXISTS task_llm_profiles (
|
||
task_id BIGINT NOT NULL REFERENCES tasks(id) ON DELETE CASCADE,
|
||
profile_id BIGINT NOT NULL REFERENCES llm_profiles(id) ON DELETE CASCADE,
|
||
position INTEGER NOT NULL CHECK (position >= 0),
|
||
status TEXT NOT NULL DEFAULT 'ready'
|
||
CHECK (status IN ('ready','quota_exhausted')),
|
||
last_error TEXT,
|
||
exhausted_at TIMESTAMPTZ,
|
||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||
updated_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||
PRIMARY KEY (task_id, profile_id),
|
||
UNIQUE (task_id, position)
|
||
);
|
||
CREATE INDEX IF NOT EXISTS idx_task_llm_profiles_order ON task_llm_profiles(task_id, position);
|
||
CREATE INDEX IF NOT EXISTS idx_task_llm_profiles_profile ON task_llm_profiles(profile_id, task_id);
|
||
CREATE INDEX IF NOT EXISTS idx_tasks_llm_profile ON tasks(llm_profile_id) WHERE llm_profile_id IS NOT NULL;
|
||
CREATE INDEX IF NOT EXISTS idx_tasks_active_llm_profile ON tasks(active_llm_profile_id) WHERE active_llm_profile_id IS NOT NULL;
|
||
DROP TRIGGER IF EXISTS trg_task_llm_profiles_upd ON task_llm_profiles;
|
||
CREATE TRIGGER trg_task_llm_profiles_upd BEFORE UPDATE ON task_llm_profiles
|
||
FOR EACH ROW EXECUTE FUNCTION set_updated_at();
|
||
|
||
-- One-time-compatible backfill: old pinned tasks become one-entry chains. A user
|
||
-- can still clear the chain later because the update path also clears the legacy
|
||
-- llm_profile_id column, preventing this block from re-adding it on restart.
|
||
INSERT INTO task_llm_profiles(task_id, profile_id, position)
|
||
SELECT t.id, t.llm_profile_id, 0
|
||
FROM tasks t
|
||
WHERE t.llm_profile_id IS NOT NULL
|
||
AND NOT EXISTS (SELECT 1 FROM task_llm_profiles x WHERE x.task_id=t.id)
|
||
ON CONFLICT DO NOTHING;
|
||
UPDATE tasks t
|
||
SET active_llm_profile_id = t.llm_profile_id
|
||
WHERE t.active_llm_profile_id IS NULL
|
||
AND t.llm_profile_id IS NOT NULL
|
||
AND EXISTS (SELECT 1 FROM task_llm_profiles x WHERE x.task_id=t.id AND x.profile_id=t.llm_profile_id);
|
||
|
||
-- 任务测试范围（资产覆盖度的分母 + 授权边界）。
|
||
-- 自动填(source='auto')：insertAssets 顶层按 worker 显式插入的资产类型加保守范围
|
||
-- （root_domain→root_domain，subdomain/service/endpoint→subdomain(host)，ip→ip）；
|
||
-- side-effect 派生的资产不入范围（钩子在 handler 顶层，派生在 db 层内部）。
|
||
-- agent 填(source='agent')：add_task_scope 加 company/root_domain/subdomain/ip。
|
||
-- 覆盖度 = 匹配 active 行的 assets（分母）中，被 fact 节点锚定过的占比（分子）。
|
||
CREATE TABLE IF NOT EXISTS task_scope (
|
||
id BIGSERIAL PRIMARY KEY,
|
||
task_id BIGINT NOT NULL REFERENCES tasks(id) ON DELETE CASCADE,
|
||
kind TEXT NOT NULL CHECK (kind IN ('company','root_domain','subdomain','ip','cidr')),
|
||
company_id BIGINT REFERENCES companies(id) ON DELETE CASCADE, -- kind='company'
|
||
domain TEXT, -- root_domain / subdomain
|
||
net CIDR, -- ip / cidr
|
||
source TEXT NOT NULL DEFAULT 'auto' CHECK (source IN ('auto','agent','manual')),
|
||
reason TEXT,
|
||
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||
);
|
||
-- 去重：同一 task 的同一条范围只存一次（自动填批量插入靠它幂等）。
|
||
CREATE UNIQUE INDEX IF NOT EXISTS uq_task_scope ON task_scope(
|
||
task_id, kind, COALESCE(domain,''), COALESCE(net::text,''), COALESCE(company_id,0));
|
||
CREATE INDEX IF NOT EXISTS idx_ts_domain ON task_scope(domain) WHERE kind IN ('root_domain','subdomain');
|
||
CREATE INDEX IF NOT EXISTS idx_ts_net ON task_scope USING GIST(net inet_ops) WHERE kind IN ('ip','cidr');
|
||
CREATE INDEX IF NOT EXISTS idx_ts_company ON task_scope(company_id) WHERE kind = 'company';
|
||
|
||
-- =====================================================================
|
||
-- E. Agents / 提示词模板 / 变量目录
|
||
-- =====================================================================
|
||
CREATE TABLE IF NOT EXISTS agents (
|
||
id BIGSERIAL PRIMARY KEY,
|
||
key TEXT NOT NULL UNIQUE CHECK (key ~ '^[a-z][a-z0-9_]*$'),
|
||
name TEXT NOT NULL,
|
||
description TEXT,
|
||
role TEXT NOT NULL,
|
||
builtin BOOLEAN NOT NULL DEFAULT true,
|
||
enabled BOOLEAN NOT NULL DEFAULT true,
|
||
llm_profile_id BIGINT REFERENCES llm_profiles(id) ON DELETE SET NULL,
|
||
current_prompt_id BIGINT,
|
||
max_turns INTEGER NOT NULL DEFAULT 0,
|
||
run_seconds INTEGER NOT NULL DEFAULT 600,
|
||
web_search BOOLEAN NOT NULL DEFAULT false,
|
||
interactive_shell BOOLEAN NOT NULL DEFAULT false,
|
||
wrapup_prompt TEXT NOT NULL DEFAULT '',
|
||
wrapup_max_turns INTEGER NOT NULL DEFAULT 0,
|
||
task_timeout_wrapup_prompt TEXT NOT NULL DEFAULT '',
|
||
task_timeout_wrapup_max_turns INTEGER NOT NULL DEFAULT 0,
|
||
trigger_run_mode TEXT NOT NULL DEFAULT 'serial' CHECK (trigger_run_mode IN ('serial','parallel')),
|
||
trigger_merge_mode TEXT NOT NULL DEFAULT 'all' CHECK (trigger_merge_mode IN ('by_task','all','none')),
|
||
trigger_max_parallel INTEGER NOT NULL DEFAULT 5,
|
||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||
updated_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||
CONSTRAINT agents_role_ck CHECK (role IN ('goals','main','planner','worker','assistant'))
|
||
);
|
||
-- 加列迁移(已发版,旧库升级补列;新库 CREATE 已含。迁移不带 CHECK:旧库存量安全 + 后端写入白名单兜底)。
|
||
ALTER TABLE agents ADD COLUMN IF NOT EXISTS trigger_run_mode TEXT NOT NULL DEFAULT 'serial';
|
||
ALTER TABLE agents ADD COLUMN IF NOT EXISTS trigger_merge_mode TEXT NOT NULL DEFAULT 'all';
|
||
ALTER TABLE agents ADD COLUMN IF NOT EXISTS trigger_max_parallel INTEGER NOT NULL DEFAULT 5;
|
||
-- per-agent LLM 绑定(agent 级默认模型):列自初版即在上方 CREATE 中,此 ALTER 仅为极旧库兜底(幂等)。
|
||
ALTER TABLE agents ADD COLUMN IF NOT EXISTS llm_profile_id BIGINT REFERENCES llm_profiles(id) ON DELETE SET NULL;
|
||
CREATE INDEX IF NOT EXISTS idx_agents_llm_profile ON agents(llm_profile_id) WHERE llm_profile_id IS NOT NULL;
|
||
DROP TRIGGER IF EXISTS trg_agents_upd ON agents;
|
||
CREATE TRIGGER trg_agents_upd BEFORE UPDATE ON agents
|
||
FOR EACH ROW EXECUTE FUNCTION set_updated_at();
|
||
|
||
CREATE TABLE IF NOT EXISTS agent_prompts (
|
||
id BIGSERIAL PRIMARY KEY,
|
||
agent_id BIGINT NOT NULL REFERENCES agents(id) ON DELETE CASCADE,
|
||
version INT NOT NULL,
|
||
template_text TEXT NOT NULL,
|
||
note TEXT,
|
||
updated_by TEXT,
|
||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||
UNIQUE (agent_id, version)
|
||
);
|
||
-- 循环外键：agents.current_prompt_id → agent_prompts.id（需在两表创建后加）
|
||
DO $$ BEGIN
|
||
ALTER TABLE agents ADD CONSTRAINT fk_agents_curprompt
|
||
FOREIGN KEY (current_prompt_id) REFERENCES agent_prompts(id) ON DELETE SET NULL;
|
||
EXCEPTION WHEN duplicate_object THEN NULL; END $$;
|
||
|
||
CREATE TABLE IF NOT EXISTS agent_prompt_vars (
|
||
id BIGSERIAL PRIMARY KEY,
|
||
agent_id BIGINT NOT NULL REFERENCES agents(id) ON DELETE CASCADE,
|
||
var_name TEXT NOT NULL,
|
||
description TEXT,
|
||
example TEXT,
|
||
source TEXT NOT NULL CHECK (source IN ('exploration','runtime','distilled')),
|
||
UNIQUE (agent_id, var_name)
|
||
);
|
||
|
||
-- =====================================================================
|
||
-- F. MCP 服务
|
||
-- =====================================================================
|
||
CREATE TABLE IF NOT EXISTS mcp_servers (
|
||
id BIGSERIAL PRIMARY KEY,
|
||
name TEXT NOT NULL UNIQUE,
|
||
transport TEXT NOT NULL CHECK (transport IN ('stdio','http')),
|
||
command TEXT,
|
||
args JSONB NOT NULL DEFAULT '[]',
|
||
env JSONB NOT NULL DEFAULT '{}',
|
||
url TEXT,
|
||
enabled BOOLEAN NOT NULL DEFAULT true,
|
||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||
updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||
);
|
||
DROP TRIGGER IF EXISTS trg_mcp_upd ON mcp_servers;
|
||
CREATE TRIGGER trg_mcp_upd BEFORE UPDATE ON mcp_servers
|
||
FOR EACH ROW EXECUTE FUNCTION set_updated_at();
|
||
|
||
-- 默认数据源占位：ScopeSentry 资产同步 MCP（地址与认证均留空、未启用）。
|
||
-- 供「资产同步」页检测数据源是否已配置；用户在页面填入 url 与 X-API-Key 后再启用。
|
||
-- 仅在缺失时插入，绝不覆盖用户已配置/已启用的服务器（schema.sql 每次启动都会 Exec）。
|
||
INSERT INTO mcp_servers (name, transport, url, env, enabled)
|
||
VALUES ('ScopeSentry', 'http', NULL, '{"X-API-Key":""}', false)
|
||
ON CONFLICT (name) DO NOTHING;
|
||
|
||
CREATE TABLE IF NOT EXISTS mcp_tools_cache (
|
||
id BIGSERIAL PRIMARY KEY,
|
||
server_id BIGINT NOT NULL REFERENCES mcp_servers(id) ON DELETE CASCADE,
|
||
tool_name TEXT NOT NULL,
|
||
description TEXT,
|
||
schema JSONB,
|
||
discovered_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||
UNIQUE (server_id, tool_name)
|
||
);
|
||
|
||
-- =====================================================================
|
||
-- G. 可见性：agent × mcp / skill
|
||
-- =====================================================================
|
||
CREATE TABLE IF NOT EXISTS agent_visibility (
|
||
agent_id BIGINT NOT NULL REFERENCES agents(id) ON DELETE CASCADE,
|
||
resource_kind TEXT NOT NULL CHECK (resource_kind IN ('mcp')),
|
||
resource_id BIGINT NOT NULL,
|
||
mcp_tool_name TEXT NOT NULL DEFAULT '',
|
||
enabled BOOLEAN NOT NULL DEFAULT true,
|
||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||
PRIMARY KEY (agent_id, resource_kind, resource_id, mcp_tool_name)
|
||
);
|
||
CREATE INDEX IF NOT EXISTS idx_vis_resource ON agent_visibility(resource_kind, resource_id);
|
||
|
||
CREATE TABLE IF NOT EXISTS agent_skill_visibility (
|
||
agent_id BIGINT NOT NULL REFERENCES agents(id) ON DELETE CASCADE,
|
||
skill_name TEXT NOT NULL,
|
||
enabled BOOLEAN NOT NULL DEFAULT true,
|
||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||
PRIMARY KEY (agent_id, skill_name)
|
||
);
|
||
CREATE INDEX IF NOT EXISTS idx_askv_skill ON agent_skill_visibility(skill_name);
|
||
|
||
-- =====================================================================
|
||
-- H. 内置工具目录
|
||
-- =====================================================================
|
||
CREATE TABLE IF NOT EXISTS tools (
|
||
key TEXT PRIMARY KEY,
|
||
system BOOLEAN NOT NULL DEFAULT true,
|
||
description TEXT NOT NULL DEFAULT '',
|
||
schema JSONB NOT NULL DEFAULT '{}',
|
||
agents JSONB NOT NULL DEFAULT '[]',
|
||
enabled BOOLEAN NOT NULL DEFAULT true,
|
||
kind TEXT NOT NULL DEFAULT 'builtin',
|
||
exec JSONB NOT NULL DEFAULT '{}',
|
||
deferred BOOLEAN NOT NULL DEFAULT false,
|
||
updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||
);
|
||
DROP TRIGGER IF EXISTS trg_tools_upd ON tools;
|
||
CREATE TRIGGER trg_tools_upd BEFORE UPDATE ON tools
|
||
FOR EACH ROW EXECUTE FUNCTION set_updated_at();
|
||
|
||
-- =====================================================================
|
||
-- I. 会话（对话页）
|
||
-- =====================================================================
|
||
CREATE TABLE IF NOT EXISTS conversations (
|
||
id BIGSERIAL PRIMARY KEY,
|
||
agent_key TEXT NOT NULL,
|
||
title TEXT NOT NULL DEFAULT '',
|
||
llm_profile_id BIGINT REFERENCES llm_profiles(id) ON DELETE SET NULL,
|
||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||
updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||
);
|
||
CREATE INDEX IF NOT EXISTS idx_conversations_llm_profile ON conversations(llm_profile_id) WHERE llm_profile_id IS NOT NULL;
|
||
DROP TRIGGER IF EXISTS trg_conversations_upd ON conversations;
|
||
CREATE TRIGGER trg_conversations_upd BEFORE UPDATE ON conversations
|
||
FOR EACH ROW EXECUTE FUNCTION set_updated_at();
|
||
|
||
CREATE TABLE IF NOT EXISTS conversation_activities (
|
||
id BIGSERIAL PRIMARY KEY,
|
||
conversation_id BIGINT NOT NULL REFERENCES conversations(id) ON DELETE CASCADE,
|
||
worker TEXT,
|
||
kind TEXT,
|
||
tool TEXT,
|
||
tool_use_id TEXT,
|
||
is_error BOOLEAN NOT NULL DEFAULT false,
|
||
summary TEXT,
|
||
detail TEXT,
|
||
input_tokens INTEGER,
|
||
output_tokens INTEGER,
|
||
cache_read_tokens INTEGER,
|
||
cache_write_tokens INTEGER,
|
||
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||
);
|
||
CREATE INDEX IF NOT EXISTS idx_conv_act ON conversation_activities(conversation_id, id);
|
||
|
||
-- =====================================================================
|
||
-- J. Agent 触发器
|
||
-- =====================================================================
|
||
CREATE TABLE IF NOT EXISTS agent_triggers (
|
||
id BIGSERIAL PRIMARY KEY,
|
||
agent_key TEXT NOT NULL,
|
||
enabled BOOLEAN NOT NULL DEFAULT true,
|
||
interval_sec INTEGER NOT NULL DEFAULT 0,
|
||
on_finding BOOLEAN NOT NULL DEFAULT false,
|
||
on_goal_met BOOLEAN NOT NULL DEFAULT false,
|
||
on_task_timeout BOOLEAN NOT NULL DEFAULT false,
|
||
on_tool_call BOOLEAN NOT NULL DEFAULT false,
|
||
on_task_create BOOLEAN NOT NULL DEFAULT false,
|
||
interval_message TEXT NOT NULL DEFAULT '',
|
||
finding_message TEXT NOT NULL DEFAULT '',
|
||
goal_message TEXT NOT NULL DEFAULT '',
|
||
task_timeout_message TEXT NOT NULL DEFAULT '',
|
||
tool_call_message TEXT NOT NULL DEFAULT '',
|
||
task_create_message TEXT NOT NULL DEFAULT '',
|
||
tool_names TEXT NOT NULL DEFAULT '',
|
||
last_fire TIMESTAMPTZ,
|
||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||
updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||
);
|
||
CREATE INDEX IF NOT EXISTS idx_agent_triggers_agent ON agent_triggers(agent_key);
|
||
-- 加列迁移(已发版,旧库升级补列;新库 CREATE 已含这些列,ALTER 为 no-op)。幂等,每次启动可重复执行。
|
||
ALTER TABLE agent_triggers ADD COLUMN IF NOT EXISTS on_tool_call BOOLEAN NOT NULL DEFAULT false;
|
||
ALTER TABLE agent_triggers ADD COLUMN IF NOT EXISTS tool_call_message TEXT NOT NULL DEFAULT '';
|
||
ALTER TABLE agent_triggers ADD COLUMN IF NOT EXISTS tool_names TEXT NOT NULL DEFAULT '';
|
||
ALTER TABLE agent_triggers ADD COLUMN IF NOT EXISTS on_task_create BOOLEAN NOT NULL DEFAULT false;
|
||
ALTER TABLE agent_triggers ADD COLUMN IF NOT EXISTS task_create_message TEXT NOT NULL DEFAULT '';
|
||
DROP TRIGGER IF EXISTS trg_agent_triggers_upd ON agent_triggers;
|
||
CREATE TRIGGER trg_agent_triggers_upd BEFORE UPDATE ON agent_triggers
|
||
FOR EACH ROW EXECUTE FUNCTION set_updated_at();
|
||
|
||
CREATE TABLE IF NOT EXISTS scheduler_state (
|
||
key TEXT PRIMARY KEY,
|
||
value TEXT NOT NULL DEFAULT ''
|
||
);
|
||
|
||
-- =====================================================================
|
||
-- K. 拦截规则
|
||
-- =====================================================================
|
||
CREATE TABLE IF NOT EXISTS intercept_rules (
|
||
id BIGSERIAL PRIMARY KEY,
|
||
name TEXT NOT NULL,
|
||
enabled BOOLEAN NOT NULL DEFAULT true,
|
||
priority INTEGER NOT NULL DEFAULT 0,
|
||
match_target TEXT NOT NULL CHECK (match_target IN ('tool_name', 'tool_input')),
|
||
match_type TEXT NOT NULL CHECK (match_type IN ('string', 'regex')),
|
||
pattern TEXT NOT NULL,
|
||
action TEXT NOT NULL CHECK (action IN ('allow', 'deny', 'ask')),
|
||
message TEXT NOT NULL DEFAULT '',
|
||
timeout_enabled BOOLEAN NOT NULL DEFAULT true,
|
||
timeout_seconds INTEGER NOT NULL DEFAULT 60,
|
||
timeout_action TEXT NOT NULL DEFAULT 'deny',
|
||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||
);
|
||
DROP TRIGGER IF EXISTS trg_intercept_rules_upd ON intercept_rules;
|
||
CREATE TRIGGER trg_intercept_rules_upd BEFORE UPDATE ON intercept_rules
|
||
FOR EACH ROW EXECUTE FUNCTION set_updated_at();
|
||
|
||
CREATE TABLE IF NOT EXISTS intercept_pending (
|
||
id BIGSERIAL PRIMARY KEY,
|
||
rule_id BIGINT REFERENCES intercept_rules(id) ON DELETE SET NULL,
|
||
conversation_id BIGINT REFERENCES conversations(id) ON DELETE CASCADE,
|
||
task_id TEXT,
|
||
agent_name TEXT NOT NULL DEFAULT '',
|
||
tool_name TEXT NOT NULL,
|
||
tool_input JSONB NOT NULL DEFAULT '{}',
|
||
status TEXT NOT NULL DEFAULT 'pending'
|
||
CHECK (status IN ('pending', 'allowed', 'denied', 'timeout')),
|
||
decided_at TIMESTAMPTZ,
|
||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||
);
|
||
CREATE INDEX IF NOT EXISTS idx_intercept_pending_status ON intercept_pending(status, created_at DESC);
|
||
CREATE INDEX IF NOT EXISTS idx_intercept_pending_task ON intercept_pending(task_id, created_at DESC);
|
||
|
||
-- =====================================================================
|
||
-- L. 漏洞发现持久化
|
||
-- =====================================================================
|
||
CREATE TABLE IF NOT EXISTS findings (
|
||
id BIGSERIAL PRIMARY KEY,
|
||
task_id BIGINT REFERENCES tasks(id) ON DELETE SET NULL,
|
||
node_id BIGINT REFERENCES exploration_nodes(id) ON DELETE SET NULL,
|
||
vulnclass TEXT NOT NULL DEFAULT '',
|
||
-- 漏洞名称(可读标题)；为空时前端回退展示 vulnclass。severity 取值：
|
||
-- critical 严重 / high 高 / medium 中 / low 低（不加 CHECK，与 status 一致由 server 白名单校验）。
|
||
name TEXT NOT NULL DEFAULT '',
|
||
severity TEXT NOT NULL DEFAULT '',
|
||
summary TEXT NOT NULL DEFAULT '',
|
||
evidence TEXT NOT NULL DEFAULT '',
|
||
worker TEXT NOT NULL DEFAULT '',
|
||
asset_ids JSONB NOT NULL DEFAULT '[]',
|
||
-- 处置状态：pending 待处理 / in_progress 处理中 / confirmed 已确认 / resolved 已处理 /
|
||
-- false_positive 误报 / ignored 忽略 / duplicate 重复 / risk_accepted 风险接受。
|
||
-- 取值不加 CHECK：旧库靠下面的 ALTER 补列,CHECK 无法回填,统一由 server 侧白名单校验。
|
||
status TEXT NOT NULL DEFAULT 'pending',
|
||
-- 漏洞详细报告(Markdown)；默认空,仅详情页读取/展示,不进列表接口以免 payload 膨胀。
|
||
report TEXT NOT NULL DEFAULT '',
|
||
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||
);
|
||
ALTER TABLE findings ADD COLUMN IF NOT EXISTS status TEXT NOT NULL DEFAULT 'pending';
|
||
ALTER TABLE findings ADD COLUMN IF NOT EXISTS name TEXT NOT NULL DEFAULT '';
|
||
ALTER TABLE findings ADD COLUMN IF NOT EXISTS report TEXT NOT NULL DEFAULT '';
|
||
CREATE INDEX IF NOT EXISTS idx_findings_task ON findings(task_id, created_at DESC);
|
||
CREATE INDEX IF NOT EXISTS idx_findings_time ON findings(created_at DESC);
|
||
CREATE INDEX IF NOT EXISTS idx_findings_status ON findings(status, created_at DESC);
|
||
|
||
-- =====================================================================
|
||
-- M. 后端日志持久化
|
||
-- =====================================================================
|
||
CREATE TABLE IF NOT EXISTS server_logs (
|
||
id BIGSERIAL PRIMARY KEY,
|
||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||
level TEXT NOT NULL DEFAULT 'info',
|
||
tag TEXT NOT NULL DEFAULT '',
|
||
text TEXT NOT NULL DEFAULT ''
|
||
);
|
||
CREATE INDEX IF NOT EXISTS idx_server_logs_id ON server_logs(id DESC);
|