From a2fa7bb10351b4a902c247e2ce7a312cd7244a1f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=AE=81=E4=BC=9F?= Date: Sun, 30 Aug 2026 22:06:45 +0800 Subject: [PATCH 1/9] =?UTF-8?q?=E8=BF=BD=E5=8A=A0http=E7=9A=84=E5=90=8C?= =?UTF-8?q?=E6=AD=A5=E8=B0=83=E7=94=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- doc/system-feature-list.md | 8 +- docker/README.md | 3 + docker/assets/scripts/startup.sh | 1 + docker/docs/environment-variables.md | 2 + .../daedalus-agent-management-web/README.md | 3 + .../docs/spec.md | 5 + .../server/src/api/routes/index.js | 19 ++ .../server/src/config/env.js | 3 + .../server/src/docs/openapi.js | 17 ++ .../server/src/services/agent-service.js | 44 +++- .../server/src/services/topic-ping-service.js | 193 ++++++++++++++++++ .../test/agent-service-alignment.test.js | 42 ++++ .../server/test/app-smoke.test.js | 48 ++++- .../server/test/env-config.test.js | 16 ++ .../server/test/topic-ping-service.test.js | 93 +++++++++ .../src/pages/AgentsPage.test.jsx | 19 +- test/prepare/stress/rebuild-stress-env.js | 1 + 17 files changed, 509 insertions(+), 8 deletions(-) diff --git a/doc/system-feature-list.md b/doc/system-feature-list.md index 5a78c4b..929b87b 100644 --- a/doc/system-feature-list.md +++ b/doc/system-feature-list.md @@ -85,6 +85,7 @@ - [x][T] Agent 工作区 Markdown 文件列表、读取、创建、保存、移动/重命名、删除、下载，使用 etag 做乐观并发校验；文件页不展示额外的“新会话生效”提示。 - [x][T] 恢复 Agent 默认 `AGENTS.md`；Agent Skill 列表、ZIP 安装/替换、删除、导出，支持 etag 和 slug 校验。 - [x][T] 管理控制台提供 Agent 入站 Topic 探测、管理 Topic 探测和对话测试；对话测试展示发送消息、回复消息、步骤耗时和完整 MQTT 报文。 +- [x][T] 为每个 Agent 暴露同步 HTTP 调用 URL；`POST /api/external/agents/{agentId}/call` 接收 `sessionId`、`senderId`、`text`，通过 MQTT 请求 Agent 并固定只返回匹配会话的 `kind=final` 最终结果（发送 `disableBlockStreaming: true`，忽略中间 `block`）。 ### 2.7 MCP 服务管理 @@ -116,7 +117,7 @@ ## 3. 外部与内部 API -- [x][T] 外部服务 API：按 `userName`/`id` 查询可用 Agent；未传身份时仅查询“所有人”可用的 Agent，并返回每项的 `is_public` 标记；支持创建/复用用户-Agent 会话、获取会话上下文、获取业务系统 Cookie、记录调用日志。 +- [x][T] 外部服务 API：按 `userName`/`id` 查询可用 Agent；未传身份时仅查询“所有人”可用的 Agent，并返回每项的 `is_public` 标记；支持创建/复用用户-Agent 会话、获取会话上下文、获取业务系统 Cookie、记录调用日志；支持 `POST /api/external/agents/{agentId}/call` 同步发送文本并只返回 `final` 最终结果。 - [x][T] 管理 Web 提供面向 `/api/external/*` 对外接口的 OpenAPI 3.0 文档和本地 Swagger UI：文档入口为 `/doc`，规范 JSON 为 `/doc/openapi.json`，包含应用场景、鉴权、参数用途/限制/可选项、请求体和常见错误说明；基础设置的版本信息下方提供“API 文档”入口。 - [x][T] 外部会话 ID 使用 `s-` 加 30 位数字格式；同一用户-Agent 复用会话；会话创建校验 Agent 状态和用户授权。 - [x][T] 外部调用在每次解析业务系统时重新校验会话有效性、Agent 状态、业务系统状态和 Agent 业务系统策略。 @@ -165,8 +166,8 @@ | 范围 | 主要验证内容 | 代表测试位置 | | --- | --- | --- | -| 管理 Web 服务 | 路由、鉴权、用户/Agent/模板/Skill/LLM/MCP/业务系统 CRUD、内置模板编辑/重命名与删除保护、模板主/备选模型和 Agent 继承、Agent 用户访问策略、Agent 多实例 MCP 选定服务同步及普通/沙箱 deny 投影、外部 Agent 目录匿名公开筛选与 `is_public` 标记、仅面向 `/api/external/*` 的 OpenAPI/Swagger 文档 5 个操作及静态资源路由、审计、同步、健康检查、日志下载、SQLite 回滚和制品安全 | `packages/daedalus-agent-management-web/server/test/app-smoke.test.js`、`packages/daedalus-agent-management-web/server/test/mcp-service.test.js`、`packages/daedalus-agent-management-web/server/management-test/mcp-sync.test.ts`、其他 `server/test/`、`server/management-test/` | -| 管理 Web 前端 | API 客户端鉴权/超时/错误归一化、同步门控、重启轮询、格式化、浏览器覆盖契约、员工与模板列表操作精简、内置模板编辑、名称/编辑详情入口、详情 tabs 保存流程、员工概览主模型/备选模型/每日配额快捷修改及打开状态下的 Token 用量同步刷新、员工技能/业务系统/MCP 与模板设置布局一致及内置 Skill 隐藏、模板文本文件切换后的编辑器可见性、设置页 API 文档入口 | `packages/daedalus-agent-management-web/src/**/*.test.js`、`packages/daedalus-agent-management-web/src/pages/AgentsPage.test.jsx`、`test/browser-coverage.test.js`、`test/run-browser-e2e.js`、`test/integration/website-full.js` | +| 管理 Web 服务 | 路由、鉴权、用户/Agent/模板/Skill/LLM/MCP/业务系统 CRUD、内置模板编辑/重命名与删除保护、模板主/备选模型和 Agent 继承、Agent 用户访问策略、Agent 多实例 MCP 选定服务同步及普通/沙箱 deny 投影、外部 Agent 目录匿名公开筛选与 `is_public` 标记、同步 Agent HTTP 接口（参数校验、服务鉴权、MQTT final-only 行为）、仅面向 `/api/external/*` 的 OpenAPI/Swagger 文档 6 个操作及静态资源路由、审计、同步、健康检查、日志下载、SQLite 回滚和制品安全 | `packages/daedalus-agent-management-web/server/test/app-smoke.test.js`、`packages/daedalus-agent-management-web/server/test/topic-ping-service.test.js`、`packages/daedalus-agent-management-web/server/test/agent-service-alignment.test.js`、`packages/daedalus-agent-management-web/server/test/env-config.test.js`、`packages/daedalus-agent-management-web/server/test/mcp-service.test.js`、`packages/daedalus-agent-management-web/server/management-test/mcp-sync.test.ts`、其他 `server/test/`、`server/management-test/` | +| 管理 Web 前端 | API 客户端鉴权/超时/错误归一化、同步门控、重启轮询、格式化、浏览器覆盖契约、员工与模板列表操作精简、内置模板编辑、名称/编辑详情入口、详情 tabs 保存流程、员工概览主模型/备选模型/每日配额快捷修改及打开状态下的 Token 用量同步刷新、员工技能/业务系统/MCP 与模板设置布局一致及内置 Skill 隐藏、编辑页不展示同步调用 URL、模板文本文件切换后的编辑器可见性、设置页 API 文档入口 | `packages/daedalus-agent-management-web/src/**/*.test.js`、`packages/daedalus-agent-management-web/src/pages/AgentsPage.test.jsx`、`test/browser-coverage.test.js`、`test/run-browser-e2e.js`、`test/integration/website-full.js` | | invoke CLI | 参数、JSON/body/query 来源、Binding schema、socket 协议、超时、大小限制、Provider 错误和调用日志 | `packages/daedalus-agent-apps-invoke-cli/test/` | | MQTT 插件 | 账号配置、旧配置兼容、环境变量覆盖、连接/断线重连、通配符、入站路由、普通消息串行队列、停止与 steer/tell 控制队列、active-run 非阻塞 steer、运行时接收 Promise 未完成时的确认与原 final 投递、原始会话目标、OpenClaw deliver 的 text/kind 无加工透传、出站 envelope、日志 | `extensions/openclaw-plugins/daedalus-agent-mqtt-channel/src/*test.ts` | | Docker 集成 | 登录、版本/同步、Provider/Model/模板/MCP/Agent 生命周期、模板继承、工作区/Skill、Ontology、MQTT `/stop`/`/steer`/`/tell` 的 final 确认及普通队列保留、重启、启动钩子、包持久化、文件输出、会话隔离和压测 | `test/case/integration/`、`test/suite/integration/`、`test/case/stress/` | @@ -175,6 +176,7 @@ ## 8. 当前边界与使用前提 - [!] 管理控制台是管理员/运维面板，不是终端对话应用；终端用户通过 CUI 或 MQTT 接入 Agent。 +- [!] 同步 Agent HTTP 接口是服务间调用接口，必须使用服务 Bearer Token（本机请求按现有服务鉴权策略处理）；它只等待并返回最终结果，调用方需要自行处理 MQTT/Agent 超时或服务不可用错误。 - [!] 业务系统当前按 HZG/活字格模型接入，Ontology ZIP 和 `index.md` 是必要前提。 - [!] Agent 只能访问配置白名单内的内网地址；白名单更新后必须重启容器。 - [!] 内置模板、内置 Skill、内置 Ontology 受删除保护；内置 Skill 和内置 Ontology 不能按普通资源直接编辑。 diff --git a/docker/README.md b/docker/README.md index b1d1402..4d5f5ba 100644 --- a/docker/README.md +++ b/docker/README.md @@ -47,12 +47,15 @@ docker start daedalus-agent ```json { "DA_MANAGEMENT_CONSOLE_PORT": 3030, + "DA_MANAGEMENT_CONSOLE_PUBLIC_BASE_URL": "https://agents.example.com", "DA_OPENCLAW_GATEWAY_MAX_CONCURRENT": 128 } ``` `DA_OPENCLAW_GATEWAY_MAX_CONCURRENT` 控制 OpenClaw gateway 执行 lane 并发数；容器启动时会写入 `/var/daedalus-agent/.openclaw/openclaw.json` 的 `agents.defaults.maxConcurrent`。 +`DA_MANAGEMENT_CONSOLE_PUBLIC_BASE_URL` 用于生成 Agent 同步 HTTP 接口 URL；未配置时使用管理控制台端口对应的 `http://127.0.0.1:`。 + 管理控制台的系统状态采用缓存刷新机制：启动后默认延迟 60 秒首次采集，之后每 60 秒刷新一次，覆盖 MQTT、S3、内核、插件加载和调用服务等状态；可通过 `DA_SYSTEM_STATUS_INITIAL_DELAY_MS`、`DA_SYSTEM_STATUS_REFRESH_INTERVAL_MS`、`DA_SYSTEM_STATUS_TIMEOUT_MS` 调整。 OpenClaw/agent 的局域网白名单通过管理控制台“系统设置 -> 基础设置”维护并保存在 SQLite；修改后重启容器生效。紧急运维时也可通过容器环境变量 `DA_INTERNAL_NETWORK_ALLOWLIST` 临时覆盖数据库值。 diff --git a/docker/assets/scripts/startup.sh b/docker/assets/scripts/startup.sh index 0afb168..0ace5d6 100644 --- a/docker/assets/scripts/startup.sh +++ b/docker/assets/scripts/startup.sh @@ -775,6 +775,7 @@ const shared = { ?? valueWithDefault("DA_INTERNAL_NETWORK_ALLOWLIST", ""), DA_OPENCLAW_GATEWAY_MAX_CONCURRENT: positiveIntegerWithDefault("DA_OPENCLAW_GATEWAY_MAX_CONCURRENT", 128), DA_MANAGEMENT_WEBSITE_TIMEOUT: String(value("DA_MANAGEMENT_WEBSITE_TIMEOUT", "180")), + DA_MANAGEMENT_CONSOLE_PUBLIC_BASE_URL: String(value("DA_MANAGEMENT_CONSOLE_PUBLIC_BASE_URL", "")), DA_SYSTEM_STATUS_INITIAL_DELAY_MS: positiveIntegerWithDefault("DA_SYSTEM_STATUS_INITIAL_DELAY_MS", 60000), DA_SYSTEM_STATUS_REFRESH_INTERVAL_MS: positiveIntegerWithDefault("DA_SYSTEM_STATUS_REFRESH_INTERVAL_MS", 60000), DA_SYSTEM_STATUS_TIMEOUT_MS: positiveIntegerWithDefault("DA_SYSTEM_STATUS_TIMEOUT_MS", 5000), diff --git a/docker/docs/environment-variables.md b/docker/docs/environment-variables.md index 9d1e43d..a69c621 100644 --- a/docker/docs/environment-variables.md +++ b/docker/docs/environment-variables.md @@ -7,6 +7,7 @@ DAEDALUS_AGENT Docker 支持通过 `/var/daedalus-agent/env.json` 配置少量 | 变量 | 默认值 | 说明 | | --- | --- | --- | | `DA_MANAGEMENT_CONSOLE_PORT` | `3030` | management web 监听端口。建议宿主机映射整个 `3030-3050` 区间。 | +| `DA_MANAGEMENT_CONSOLE_PUBLIC_BASE_URL` | 空（回退 `http://127.0.0.1:`） | 对外同步 Agent HTTP 接口生成 URL 使用的管理控制台基址；应为 `http` 或 `https` URL，末尾斜杠可省略，非法值按回退地址处理。 | | `DA_OPENCLAW_GATEWAY_MAX_CONCURRENT` | `128` | OpenClaw gateway 执行 lane 并发数，启动时写入 `openclaw.json` 的 `agents.defaults.maxConcurrent`。 | | `DA_MANAGEMENT_WEBSITE_TIMEOUT` | `180` | management web 访问内部管理动作的超时时间，单位秒。 | | `DA_SYSTEM_STATUS_INITIAL_DELAY_MS` | `60000` | 管理控制台启动后首次采集系统状态的延迟，单位毫秒；覆盖 MQTT、S3、内核、插件加载和调用服务等状态。 | @@ -18,6 +19,7 @@ DAEDALUS_AGENT Docker 支持通过 `/var/daedalus-agent/env.json` 配置少量 ```json { "DA_MANAGEMENT_CONSOLE_PORT": 3030, + "DA_MANAGEMENT_CONSOLE_PUBLIC_BASE_URL": "https://agents.example.com", "DA_OPENCLAW_GATEWAY_MAX_CONCURRENT": 128, "DA_SYSTEM_STATUS_INITIAL_DELAY_MS": 60000, "DA_SYSTEM_STATUS_REFRESH_INTERVAL_MS": 60000, diff --git a/packages/daedalus-agent-management-web/README.md b/packages/daedalus-agent-management-web/README.md index f0dcef7..d3bcdd6 100644 --- a/packages/daedalus-agent-management-web/README.md +++ b/packages/daedalus-agent-management-web/README.md @@ -103,9 +103,12 @@ npm run test:full - `GET /api/external/agents`（未传 `userName`/`id` 时仅返回“所有人”可用的 Agent） - `GET /api/external/agents?userName=zhangsan`（也支持使用 `id` 作为身份参数） +- `POST /api/external/agents/:agentId/call`（请求体为 `sessionId`、`senderId`、`text`；固定只返回 `kind=final` 最终结果） - `GET /api/external/session?userName=zhangsan&agentId=finance-agent` - `GET /api/external/context?sessionId=...` +Agent 目录项提供该同步接口的 URL。调用需要服务 Bearer Token；服务端通过 Agent MQTT 入站/出站 Topic 转发请求，并固定发送 `disableBlockStreaming: true`，忽略中间 `block` 消息。 + API 文档： - `GET /doc`：本地 Swagger UI 文档页面。 diff --git a/packages/daedalus-agent-management-web/docs/spec.md b/packages/daedalus-agent-management-web/docs/spec.md index ed8b070..c04abd9 100644 --- a/packages/daedalus-agent-management-web/docs/spec.md +++ b/packages/daedalus-agent-management-web/docs/spec.md @@ -11,6 +11,7 @@ - 模板、Skill、LLM、业务系统的管理 UI - 本地状态、日志、统计和同步状态展示 - CUI / invoke CLI service API +- Agent 同步 HTTP 调用 API OpenClaw 侧真实目录通过同步任务写入本地数据库。页面读取数据库，不在页面加载时直接打多个管理动作查询。 @@ -30,6 +31,10 @@ OpenClaw 侧真实目录通过同步任务写入本地数据库。页面读取 Token 用量由 `agent.usage.list` 同步到 `agents.usage_snapshot_json`。超过每日限额后状态可变为 `overlimit`，并从 CUI 可用目录中排除。 +### Agent 同步调用 + +`POST /api/external/agents/{agentId}/call` 为指定 Agent 提供同步文本调用，JSON 请求体固定包含 `sessionId`、`senderId` 和 `text`。控制台将请求转发到 Agent MQTT 入站 Topic，并设置 `disableBlockStreaming: true`；只等待匹配 `sessionId` 的 `kind: "final"` 消息，忽略中间 `block` 消息后返回最终结果。接口使用服务 Bearer Token 鉴权，本机请求遵循现有服务鉴权例外。 + ### 模板 模板由模板名和 workspace zip 组成，zip 顶层必须包含 `AGENTS.md`，并应保留可复用目录，例如 `.learnings/`、`skills/`、`knowledge/`。模板可配置默认 Provider / Model，创建数字员工时作为表单默认值。 diff --git a/packages/daedalus-agent-management-web/server/src/api/routes/index.js b/packages/daedalus-agent-management-web/server/src/api/routes/index.js index d94f6fd..f18a991 100644 --- a/packages/daedalus-agent-management-web/server/src/api/routes/index.js +++ b/packages/daedalus-agent-management-web/server/src/api/routes/index.js @@ -341,6 +341,25 @@ export function createRoutes(services) { }); })); + router.post("/external/agents/:agentId/call", requireServiceApiAuth, asyncRoute(async (req, res) => { + const payload = req.body || {}; + if (!String(payload.sessionId || "").trim()) { + throw badRequest("sessionId 不能为空"); + } + if (!String(payload.senderId || "").trim()) { + throw badRequest("senderId 不能为空"); + } + if (!String(payload.text || "").trim()) { + throw badRequest("文本不能为空"); + } + const result = await services.topicPingService.syncAgentMessage(req.params.agentId, { + sessionId: payload.sessionId, + senderId: payload.senderId, + text: payload.text + }); + res.json(result); + })); + router.get("/external/session", requireServiceApiAuth, asyncRoute(async (req, res) => { const result = await services.externalService.getOrCreateSession({ userName: req.query.userName, diff --git a/packages/daedalus-agent-management-web/server/src/config/env.js b/packages/daedalus-agent-management-web/server/src/config/env.js index a712a46..4b9c4ba 100644 --- a/packages/daedalus-agent-management-web/server/src/config/env.js +++ b/packages/daedalus-agent-management-web/server/src/config/env.js @@ -85,6 +85,9 @@ export function loadEnv() { return { port: readEnvNumber("DA_WEB_PORT", defaultPort), + http: { + publicBaseUrl: readEnvValue("DA_MANAGEMENT_CONSOLE_PUBLIC_BASE_URL") + }, runMode: readEnvValue("DA_RUN_MODE", ""), dataDir, localArtifactsDir, diff --git a/packages/daedalus-agent-management-web/server/src/docs/openapi.js b/packages/daedalus-agent-management-web/server/src/docs/openapi.js index e45252e..fcf3d43 100644 --- a/packages/daedalus-agent-management-web/server/src/docs/openapi.js +++ b/packages/daedalus-agent-management-web/server/src/docs/openapi.js @@ -149,6 +149,11 @@ const agentBody = jsonBody("数字员工配置。创建时需要 id/agent_name/t tags: { type: "array", items: { type: "string" }, description: "标签列表，可选。" }, restart: booleanField("是否在远端变更后重启运行时；默认按服务端策略执行。") }); +const syncAgentMessageBody = jsonBody("同步 Agent 请求。服务端通过 MQTT 等待最终回复，不向调用方返回中间 block。", { + sessionId: textField("本次请求的会话 ID；用于关联 MQTT 回复。", { minLength: 1 }), + senderId: textField("发送者 ID；用于 Agent 会话路由。", { minLength: 1 }), + text: textField("发送给 Agent 的文本；不能为空。", { minLength: 1 }) +}, ["sessionId", "senderId", "text"]); const settingsBody = jsonBody("门户基础设置。", { portal_name: textField("门户名称，不能为空。"), brand_subtitle: textField("门户副标题，不能为空。"), @@ -229,6 +234,12 @@ const ROUTES = [ ], successSchema: { type: "object", properties: { items: { type: "array", items: { $ref: "#/components/schemas/ExternalAgentDirectoryItem" } } } } }), + route("post", "/external/agents/:agentId/call", "外部服务", "同步调用 Agent", "向指定数字员工发送一条文本消息并等待最终结果。服务端固定关闭 block 流式返回，只返回匹配请求会话的 `kind=final` 消息。", { + authDescription: "需要服务 API Bearer Token。", + body: syncAgentMessageBody, + successSchema: { $ref: "#/components/schemas/ExternalAgentSyncResponse" }, + errors: { 503: "MQTT 连接、发送或等待最终回复失败。" } + }), route("get", "/external/session", "外部服务", "创建或复用外部会话", "按用户与 Agent 复用会话；创建时校验 Agent 状态、用户授权和必要 Cookie。", { authDescription: "需要服务 API Bearer Token。", query: [ @@ -614,6 +625,7 @@ const ROUTES = [ const PARAMETER_DESCRIPTIONS = { id: "资源数字 ID；必须是正整数，且必须对应现存资源。", + agentId: "Agent slug；必须是现存数字员工的 slug。", slug: "Agent slug；仅允许小写字母、数字和中划线，且不能以中划线开头或结尾。", templateName: "模板名称；必须是现存模板名称，路径参数会按 URL 编码解析。", providerId: "模型 Provider ID；必须是现存 Provider。", @@ -729,6 +741,11 @@ export const OPENAPI_DOCUMENT = { users: { type: "array", items: { type: "string" }, description: "当前目录项关联的用户列表。" }, is_public: booleanField("是否为“所有人”可用的 Agent。") }, "外部 Agent 目录项。"), + ExternalAgentSyncResponse: objectSchema({ + sessionId: textField("请求会话 ID。"), + kind: textField("固定为 final；不会返回 block 中间结果。", { enum: ["final"] }), + text: textField("Agent 最终回复文本。") + }, "同步 Agent 调用的最终结果。"), Pagination: objectSchema({ page: integerField("当前页码。", { minimum: 1 }), pageSize: integerField("每页条数。", { minimum: 1 }), diff --git a/packages/daedalus-agent-management-web/server/src/services/agent-service.js b/packages/daedalus-agent-management-web/server/src/services/agent-service.js index 5532def..2a55a48 100644 --- a/packages/daedalus-agent-management-web/server/src/services/agent-service.js +++ b/packages/daedalus-agent-management-web/server/src/services/agent-service.js @@ -79,6 +79,41 @@ function renderTopic(template, agentId) { return String(template || "").replaceAll("{agentId}", agentId); } +function resolvePublicBaseUrl(env) { + const configured = firstText(env?.http?.publicBaseUrl, env?.publicBaseUrl); + const fallbackPort = Number.isFinite(Number(env?.port)) && Number(env.port) > 0 + ? Number(env.port) + : 3030; + const fallback = `http://127.0.0.1:${fallbackPort}`; + if (!configured) { + return fallback; + } + + try { + const parsed = new URL(configured); + if (parsed.protocol !== "http:" && parsed.protocol !== "https:") { + return fallback; + } + return configured.replace(/\/+$/, ""); + } catch { + return fallback; + } +} + +function buildAgentIntegration(agent, remote, env) { + const baseUrl = resolvePublicBaseUrl(env); + const { inboundTopic, outboundTopic } = extractAgentTopics(remote, agent.slug, env); + return { + mqtt: { + inbound_topic: inboundTopic, + outbound_topic: outboundTopic + }, + http: { + endpoint_url: `${baseUrl}/api/external/agents/${encodeURIComponent(agent.slug)}/call` + } + }; +} + function sanitizeFileName(fileName) { return String(fileName || "agent").replace(/[^A-Za-z0-9._-]+/g, "-"); } @@ -156,7 +191,7 @@ function computeHealth(agent, usage) { return "normal"; } -function extractAgentTopics(remote) { +function extractAgentTopics(remote, agentId = "", env = null) { const topics = remote?.topics && typeof remote.topics === "object" ? remote.topics : {}; return { inboundTopic: firstText( @@ -164,14 +199,16 @@ function extractAgentTopics(remote) { remote?.["inbound-topic"], topics.inbound, topics.inboundTopic, - topics["inbound-topic"] + topics["inbound-topic"], + agentId && renderTopic(env?.mqtt?.agentInboundTopicTemplate, agentId) ), outboundTopic: firstText( remote?.outboundTopic, remote?.["outbound-topic"], topics.outbound, topics.outboundTopic, - topics["outbound-topic"] + topics["outbound-topic"], + agentId && renderTopic(env?.mqtt?.agentOutboundTopicTemplate, agentId) ) }; } @@ -714,6 +751,7 @@ export class AgentService { llm_model: row.llm_model_ref ? this.getLlmModelByRef(row.llm_model_ref) || null : null, fallback_llm_model: row.fallback_llm_model_ref ? this.getLlmModelByRef(row.fallback_llm_model_ref) || null : null, remote_state: remote, + integration: buildAgentIntegration(row, remote, this.env), health: computeHealth(row, usage) }; }); diff --git a/packages/daedalus-agent-management-web/server/src/services/topic-ping-service.js b/packages/daedalus-agent-management-web/server/src/services/topic-ping-service.js index 55f0478..9664f3e 100644 --- a/packages/daedalus-agent-management-web/server/src/services/topic-ping-service.js +++ b/packages/daedalus-agent-management-web/server/src/services/topic-ping-service.js @@ -6,6 +6,7 @@ import mqtt from "mqtt"; import { badRequest, notFound, serviceUnavailable } from "../utils/errors.js"; const DEFAULT_TIMEOUT_MS = 15000; +const SYNC_MESSAGE_TIMEOUT_MS = 120000; const BLOCK_STREAM_IDLE_MS = 1500; function firstText(...values) { @@ -353,6 +354,198 @@ export class TopicPingService { } } + async syncAgentMessage(agentId, { + sessionId, + senderId, + text, + timeoutMs = SYNC_MESSAGE_TIMEOUT_MS + } = {}) { + const normalizedSessionId = String(sessionId || "").trim(); + const normalizedSenderId = String(senderId || "").trim(); + const normalizedText = String(text || "").trim(); + if (!normalizedSessionId) { + throw badRequest("sessionId 不能为空"); + } + if (!normalizedSenderId) { + throw badRequest("senderId 不能为空"); + } + if (!normalizedText) { + throw badRequest("文本不能为空"); + } + if (!this.isConfigured()) { + throw serviceUnavailable("MQTT broker 未配置"); + } + + const normalizedAgentId = String(agentId || "").trim(); + const { inboundTopic, outboundTopic } = this.resolveAgentTopics(normalizedAgentId); + const safeTimeoutMs = normalizeTimeoutMs(timeoutMs, SYNC_MESSAGE_TIMEOUT_MS); + const payload = { + sessionId: normalizedSessionId, + senderId: normalizedSenderId, + disableBlockStreaming: true, + text: normalizedText + }; + const connect = this.mqttFactory.connect || this.mqttFactory; + let client; + try { + client = connect(this.env.mqtt.brokerUrl, { + clientId: `daedalus-agent-web-agent-sync-${randomUUID().slice(0, 8)}`, + username: this.env.mqtt.username || undefined, + password: this.env.mqtt.password || undefined, + clean: true, + reconnectPeriod: 0 + }); + } catch (error) { + throw serviceUnavailable("连接 MQTT broker 失败", { + cause: error?.message || String(error) + }); + } + + let cancelReplyWait = () => {}; + try { + await new Promise((resolve, reject) => { + const timer = setTimeout(() => { + cleanup(); + reject(serviceUnavailable("连接 MQTT broker 超时", { + timeoutMs: safeTimeoutMs + })); + }, safeTimeoutMs); + const cleanup = () => { + clearTimeout(timer); + client.off("connect", onConnect); + client.off("error", onError); + }; + const onConnect = () => { + cleanup(); + resolve(); + }; + const onError = (error) => { + cleanup(); + reject(serviceUnavailable("连接 MQTT broker 失败", { + cause: error?.message || String(error) + })); + }; + client.once("connect", onConnect); + client.once("error", onError); + }); + + await new Promise((resolve, reject) => { + const timer = setTimeout(() => { + reject(serviceUnavailable("订阅数字员工 MQTT 出站主题超时", { + timeoutMs: safeTimeoutMs, + outboundTopic + })); + }, safeTimeoutMs); + const done = (error) => { + clearTimeout(timer); + if (error) { + reject(serviceUnavailable("订阅数字员工 MQTT 出站主题失败", { + cause: error?.message || String(error), + outboundTopic + })); + return; + } + resolve(); + }; + try { + client.subscribe(outboundTopic, { qos: 1 }, done); + } catch (error) { + done(error); + } + }); + + const replyPromise = new Promise((resolve, reject) => { + const timer = setTimeout(() => { + cleanup(); + reject(serviceUnavailable(`等待数字员工 ${normalizedAgentId} 最终回复超时`, { + timeoutMs: safeTimeoutMs, + sessionId: normalizedSessionId, + inboundTopic, + outboundTopic + })); + }, safeTimeoutMs); + const cleanup = () => { + clearTimeout(timer); + client.off("message", onMessage); + }; + const onMessage = (topic, raw) => { + if (topic !== outboundTopic) { + return; + } + + let message; + try { + message = JSON.parse(raw.toString("utf8")); + } catch { + return; + } + + if ( + !message + || typeof message !== "object" + || String(message.sessionId || "").trim() !== normalizedSessionId + || message.kind !== "final" + ) { + return; + } + + cleanup(); + resolve(message); + }; + cancelReplyWait = () => { + cleanup(); + reject(serviceUnavailable("已停止等待数字员工 MQTT 最终回复", { + sessionId: normalizedSessionId, + inboundTopic, + outboundTopic + })); + }; + client.on("message", onMessage); + }); + // A publish failure can reject the waiter before the outer await observes it. + replyPromise.catch(() => {}); + + await new Promise((resolve, reject) => { + const timer = setTimeout(() => { + cancelReplyWait(); + reject(serviceUnavailable("发送同步消息超时", { + timeoutMs: safeTimeoutMs, + inboundTopic, + outboundTopic + })); + }, safeTimeoutMs); + const handlePublishResult = (error) => { + clearTimeout(timer); + if (error) { + cancelReplyWait(); + reject(serviceUnavailable("发送同步消息失败", { + cause: error?.message || String(error), + inboundTopic, + outboundTopic + })); + return; + } + resolve(); + }; + try { + client.publish(inboundTopic, JSON.stringify(payload), { qos: 1, retain: false }, handlePublishResult); + } catch (error) { + handlePublishResult(error); + } + }); + + return await replyPromise; + } finally { + await new Promise((resolve) => { + try { + client.end(false, {}, () => resolve()); + } catch { + resolve(); + } + }); + } + } + async waitForAgentReady(agentId, { timeoutMs = 90000, intervalMs = 2000, diff --git a/packages/daedalus-agent-management-web/server/test/agent-service-alignment.test.js b/packages/daedalus-agent-management-web/server/test/agent-service-alignment.test.js index f2de766..db1cec4 100644 --- a/packages/daedalus-agent-management-web/server/test/agent-service-alignment.test.js +++ b/packages/daedalus-agent-management-web/server/test/agent-service-alignment.test.js @@ -127,6 +127,48 @@ it("reports normal health for normal agents", async () => { expect(agent.health).toBe("normal"); }); +it("exposes MQTT topics and the synchronous HTTP endpoint for each agent", async () => { + const service = new AgentService({ + db: createDb({ + id: 4, + slug: "finance-agent", + agent_name: "Finance Agent", + description: "", + docs_content: "", + template_name: "default", + status: "normal", + tags_json: "[]", + daily_limit: -1, + usage_snapshot_json: "{}", + remote_state_json: JSON.stringify({ + inboundTopic: "agent/finance/in", + outboundTopic: "agent/finance/out" + }), + created_at: "2026-05-26T00:00:00.000Z", + updated_at: "2026-05-26T00:00:00.000Z" + }), + managementClient: {}, + objectStorage: {}, + env: { + port: 3030, + http: { + publicBaseUrl: "https://agents.example.test/" + } + } + }); + + const [agent] = await service.listAgents(); + expect(agent.integration).toEqual({ + mqtt: { + inbound_topic: "agent/finance/in", + outbound_topic: "agent/finance/out" + }, + http: { + endpoint_url: "https://agents.example.test/api/external/agents/finance-agent/call" + } + }); +}); + it("rejects setting an over-quota agent back to normal", async () => { const agentRow = { id: 6, diff --git a/packages/daedalus-agent-management-web/server/test/app-smoke.test.js b/packages/daedalus-agent-management-web/server/test/app-smoke.test.js index 36d348a..5f907f0 100644 --- a/packages/daedalus-agent-management-web/server/test/app-smoke.test.js +++ b/packages/daedalus-agent-management-web/server/test/app-smoke.test.js @@ -42,7 +42,7 @@ it("assembles the management server and maps application and parser errors", asy expect(docsJson.openapi).toBe("3.0.3"); const documentedOperationCount = Object.values(docsJson.paths) .reduce((count, pathItem) => count + Object.keys(pathItem).length, 0); - expect(documentedOperationCount).toBe(5); + expect(documentedOperationCount).toBe(6); expect(Object.keys(docsJson.paths).every((pathName) => pathName.startsWith("/external/"))).toBe(true); expect(docsJson.paths["/external/agents"].get.parameters).toEqual( jasmine.arrayContaining([ @@ -53,6 +53,9 @@ it("assembles the management server and maps application and parser errors", asy expect(docsJson.components.schemas.ExternalAgentDirectoryItem.properties.is_public).toEqual( jasmine.objectContaining({ type: "boolean" }) ); + expect(docsJson.paths["/external/agents/{agentId}/call"].post.requestBody.content["application/json"].schema.required) + .toEqual(["sessionId", "senderId", "text"]); + expect(docsJson.components.schemas.ExternalAgentSyncResponse.properties.kind.enum).toEqual(["final"]); const badLogin = await fetch(`${baseUrl}/api/auth/login`, { method: "POST", @@ -110,6 +113,49 @@ it("supports anonymous public external agent directories and id identity aliases } }); +it("handles synchronous agent requests and returns only the final result", async () => { + const { app, services } = createServerApp(); + const calls = []; + services.topicPingService.syncAgentMessage = async (agentId, payload) => { + calls.push({ agentId, payload }); + return { + sessionId: payload.sessionId, + kind: "final", + text: "同步结果" + }; + }; + + const server = await listen(app); + const baseUrl = `http://127.0.0.1:${server.address().port}`; + + try { + const response = await fetch(`${baseUrl}/api/external/agents/finance-agent/call`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ sessionId: "session-1", senderId: "sender-1", text: "hello" }) + }); + expect(response.status).toBe(200); + expect(await response.json()).toEqual({ + sessionId: "session-1", + kind: "final", + text: "同步结果" + }); + expect(calls).toEqual([{ + agentId: "finance-agent", + payload: { sessionId: "session-1", senderId: "sender-1", text: "hello" } + }]); + + const invalidResponse = await fetch(`${baseUrl}/api/external/agents/finance-agent/call`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ sessionId: "session-1", senderId: "sender-1" }) + }); + expect(invalidResponse.status).toBe(400); + } finally { + await close(server); + } +}); + it("serves the frontend bundle when a dist directory exists", async () => { const originalDirectory = process.cwd(); const fixtureDirectory = fs.mkdtempSync(path.join(os.tmpdir(), "daedalus-app-dist-")); diff --git a/packages/daedalus-agent-management-web/server/test/env-config.test.js b/packages/daedalus-agent-management-web/server/test/env-config.test.js index c59a2ba..2bcd49a 100644 --- a/packages/daedalus-agent-management-web/server/test/env-config.test.js +++ b/packages/daedalus-agent-management-web/server/test/env-config.test.js @@ -257,3 +257,19 @@ it("falls back to management console port for the web server", () => { } } }); + +it("reads the public management base URL for generated agent endpoints", () => { + const originalPublicBaseUrl = process.env.DA_MANAGEMENT_CONSOLE_PUBLIC_BASE_URL; + process.env.DA_MANAGEMENT_CONSOLE_PUBLIC_BASE_URL = "https://agents.example.test/"; + + try { + const env = loadEnv(); + expect(env.http.publicBaseUrl).toBe("https://agents.example.test/"); + } finally { + if (originalPublicBaseUrl === undefined) { + delete process.env.DA_MANAGEMENT_CONSOLE_PUBLIC_BASE_URL; + } else { + process.env.DA_MANAGEMENT_CONSOLE_PUBLIC_BASE_URL = originalPublicBaseUrl; + } + } +}); diff --git a/packages/daedalus-agent-management-web/server/test/topic-ping-service.test.js b/packages/daedalus-agent-management-web/server/test/topic-ping-service.test.js index 5dd8450..c693917 100644 --- a/packages/daedalus-agent-management-web/server/test/topic-ping-service.test.js +++ b/packages/daedalus-agent-management-web/server/test/topic-ping-service.test.js @@ -278,6 +278,99 @@ it("sends a single agent test message and returns MQTT payloads with timings", a expect(result.timings.total_ms).toEqual(jasmine.any(Number)); }); +it("sends a synchronous agent message with block streaming disabled and returns only final", async () => { + const client = new MockMqttClient(); + const service = new TopicPingService({ + db: { + prepare() { + return { + get() { + return { + slug: "agent-a", + remote_state_json: JSON.stringify({ + inboundTopic: "agent/agent-a/in", + outboundTopic: "agent/agent-a/out" + }) + }; + } + }; + } + }, + env: { + mqtt: { + brokerUrl: "mqtt://broker", + agentInboundTopicTemplate: "agent/{agentId}/in", + agentOutboundTopicTemplate: "agent/{agentId}/out" + } + }, + mqttFactory: { + connect() { + queueMicrotask(() => client.emit("connect")); + return client; + } + } + }); + + client.publish = function publish(topic, payload, _options, callback) { + this.publications.push({ topic, payload: JSON.parse(payload) }); + callback?.(null); + queueMicrotask(() => { + const sessionId = this.publications[0].payload.sessionId; + this.emit("message", this.subscriptions[0], Buffer.from(JSON.stringify({ + sessionId, + kind: "block", + text: "intermediate" + }), "utf8")); + this.emit("message", this.subscriptions[0], Buffer.from(JSON.stringify({ + sessionId, + kind: "final", + text: "final answer" + }), "utf8")); + }); + }; + + const result = await service.syncAgentMessage("agent-a", { + sessionId: "session-1", + senderId: "sender-1", + text: "hello" + }); + + expect(result).toEqual({ + sessionId: "session-1", + kind: "final", + text: "final answer" + }); + expect(client.publications[0]).toEqual({ + topic: "agent/agent-a/in", + payload: { + sessionId: "session-1", + senderId: "sender-1", + disableBlockStreaming: true, + text: "hello" + } + }); +}); + +it("validates synchronous agent message fields", async () => { + const service = createTopicService(); + + await expectAsync(service.syncAgentMessage("agent-a", { + sessionId: "", + senderId: "sender-1", + text: "hello" + })).toBeRejectedWithError(/sessionId/); + await expectAsync(service.syncAgentMessage("agent-a", { + sessionId: "session-1", + senderId: "", + text: "hello" + })).toBeRejectedWithError(/senderId/); + await expectAsync(service.syncAgentMessage("agent-a", { + sessionId: "session-1", + senderId: "sender-1", + text: " " + })).toBeRejectedWithError(/文本/); +}); + it("returns an empty reply list when agent test fails before waiting for replies", async () => { const client = new MockMqttClient(); const service = new TopicPingService({ diff --git a/packages/daedalus-agent-management-web/src/pages/AgentsPage.test.jsx b/packages/daedalus-agent-management-web/src/pages/AgentsPage.test.jsx index 5caf7aa..0d722cf 100644 --- a/packages/daedalus-agent-management-web/src/pages/AgentsPage.test.jsx +++ b/packages/daedalus-agent-management-web/src/pages/AgentsPage.test.jsx @@ -41,7 +41,12 @@ const agent = { daily_limit: 10000, usage: { daily: 1200 }, permissions: [], - user_policy: { mode: "selected" } + user_policy: { mode: "selected" }, + integration: { + http: { + endpoint_url: "https://agents.example.test/api/external/agents/finance-agent/call" + } + } }; function setupApi(getAgent = () => agent) { @@ -60,6 +65,18 @@ function setupApi(getAgent = () => agent) { } describe("AgentsPage overview quick edits", () => { + it("does not show the synchronous HTTP endpoint in edit details", async () => { + setupApi(); + render(); + + await waitFor(() => expect(screen.getByTestId("agent-edit-finance-agent")).toBeInTheDocument()); + fireEvent.click(screen.getAllByTestId("agent-edit-finance-agent")[0]); + + await screen.findByText("入站主题"); + expect(screen.queryByText("同步调用 URL")).not.toBeInTheDocument(); + expect(screen.queryByText("https://agents.example.test/api/external/agents/finance-agent/call")).not.toBeInTheDocument(); + }); + it("shows separate model and daily quota edit actions in edit mode", async () => { setupApi(); render(); diff --git a/test/prepare/stress/rebuild-stress-env.js b/test/prepare/stress/rebuild-stress-env.js index d616fac..fbb370a 100644 --- a/test/prepare/stress/rebuild-stress-env.js +++ b/test/prepare/stress/rebuild-stress-env.js @@ -439,6 +439,7 @@ function buildRuntimeEnv(mqttPassword, minioPassword, agentNetworkAllowlist = "" DA_WEB_DATA_DIR: "/var/daedalus-agent/management/data", DA_INTERNAL_NETWORK_ALLOWLIST: agentNetworkAllowlist, DA_MANAGEMENT_WEBSITE_TIMEOUT: "180", + DA_MANAGEMENT_CONSOLE_PUBLIC_BASE_URL: "", DA_MANAGEMENT_CONSOLE_PORT: "3030", PORT: "3030" }; -- Gitee From f8a1f443b67bbdaede619cde067e294b32ffd0de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=AE=81=E4=BC=9F?= Date: Mon, 31 Aug 2026 20:31:59 +0800 Subject: [PATCH 2/9] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E5=90=8C=E6=AD=A5?= =?UTF-8?q?=E8=B0=83=E7=94=A8=E6=8E=A5=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- doc/system-feature-list.md | 8 ++-- .../daedalus-agent-management-web/README.md | 2 +- .../docs/spec.md | 2 +- .../server/src/api/routes/index.js | 25 ++++++++++++ .../server/src/docs/openapi.js | 2 +- .../server/test/app-smoke.test.js | 38 +++++++++++++++++++ 6 files changed, 70 insertions(+), 7 deletions(-) diff --git a/doc/system-feature-list.md b/doc/system-feature-list.md index 929b87b..5b624ff 100644 --- a/doc/system-feature-list.md +++ b/doc/system-feature-list.md @@ -85,7 +85,7 @@ - [x][T] Agent 工作区 Markdown 文件列表、读取、创建、保存、移动/重命名、删除、下载，使用 etag 做乐观并发校验；文件页不展示额外的“新会话生效”提示。 - [x][T] 恢复 Agent 默认 `AGENTS.md`；Agent Skill 列表、ZIP 安装/替换、删除、导出，支持 etag 和 slug 校验。 - [x][T] 管理控制台提供 Agent 入站 Topic 探测、管理 Topic 探测和对话测试；对话测试展示发送消息、回复消息、步骤耗时和完整 MQTT 报文。 -- [x][T] 为每个 Agent 暴露同步 HTTP 调用 URL；`POST /api/external/agents/{agentId}/call` 接收 `sessionId`、`senderId`、`text`，通过 MQTT 请求 Agent 并固定只返回匹配会话的 `kind=final` 最终结果（发送 `disableBlockStreaming: true`，忽略中间 `block`）。 +- [x][T] 为每个 Agent 暴露同步 HTTP 调用 URL；`POST /api/external/agents/{agentId}/call` 接收 `sessionId`、`senderId`、`text`，通过 MQTT 请求 Agent 并固定只返回匹配会话的 `kind=final` 最终结果（发送 `disableBlockStreaming: true`，忽略中间 `block`）；成功后刷新 Agent 用量快照，将本次调用产生的 Token 计入每日配额并更新超额状态。 ### 2.7 MCP 服务管理 @@ -117,7 +117,7 @@ ## 3. 外部与内部 API -- [x][T] 外部服务 API：按 `userName`/`id` 查询可用 Agent；未传身份时仅查询“所有人”可用的 Agent，并返回每项的 `is_public` 标记；支持创建/复用用户-Agent 会话、获取会话上下文、获取业务系统 Cookie、记录调用日志；支持 `POST /api/external/agents/{agentId}/call` 同步发送文本并只返回 `final` 最终结果。 +- [x][T] 外部服务 API：按 `userName`/`id` 查询可用 Agent；未传身份时仅查询“所有人”可用的 Agent，并返回每项的 `is_public` 标记；支持创建/复用用户-Agent 会话、获取会话上下文、获取业务系统 Cookie、记录调用日志；支持 `POST /api/external/agents/{agentId}/call` 同步发送文本并只返回 `final` 最终结果，成功后刷新用量并计入每日配额。 - [x][T] 管理 Web 提供面向 `/api/external/*` 对外接口的 OpenAPI 3.0 文档和本地 Swagger UI：文档入口为 `/doc`，规范 JSON 为 `/doc/openapi.json`，包含应用场景、鉴权、参数用途/限制/可选项、请求体和常见错误说明；基础设置的版本信息下方提供“API 文档”入口。 - [x][T] 外部会话 ID 使用 `s-` 加 30 位数字格式；同一用户-Agent 复用会话；会话创建校验 Agent 状态和用户授权。 - [x][T] 外部调用在每次解析业务系统时重新校验会话有效性、Agent 状态、业务系统状态和 Agent 业务系统策略。 @@ -166,7 +166,7 @@ | 范围 | 主要验证内容 | 代表测试位置 | | --- | --- | --- | -| 管理 Web 服务 | 路由、鉴权、用户/Agent/模板/Skill/LLM/MCP/业务系统 CRUD、内置模板编辑/重命名与删除保护、模板主/备选模型和 Agent 继承、Agent 用户访问策略、Agent 多实例 MCP 选定服务同步及普通/沙箱 deny 投影、外部 Agent 目录匿名公开筛选与 `is_public` 标记、同步 Agent HTTP 接口（参数校验、服务鉴权、MQTT final-only 行为）、仅面向 `/api/external/*` 的 OpenAPI/Swagger 文档 6 个操作及静态资源路由、审计、同步、健康检查、日志下载、SQLite 回滚和制品安全 | `packages/daedalus-agent-management-web/server/test/app-smoke.test.js`、`packages/daedalus-agent-management-web/server/test/topic-ping-service.test.js`、`packages/daedalus-agent-management-web/server/test/agent-service-alignment.test.js`、`packages/daedalus-agent-management-web/server/test/env-config.test.js`、`packages/daedalus-agent-management-web/server/test/mcp-service.test.js`、`packages/daedalus-agent-management-web/server/management-test/mcp-sync.test.ts`、其他 `server/test/`、`server/management-test/` | +| 管理 Web 服务 | 路由、鉴权、用户/Agent/模板/Skill/LLM/MCP/业务系统 CRUD、内置模板编辑/重命名与删除保护、模板主/备选模型和 Agent 继承、Agent 用户访问策略、Agent 多实例 MCP 选定服务同步及普通/沙箱 deny 投影、外部 Agent 目录匿名公开筛选与 `is_public` 标记、同步 Agent HTTP 接口（参数校验、服务鉴权、MQTT final-only 行为、成功后刷新用量并计入每日配额）、仅面向 `/api/external/*` 的 OpenAPI/Swagger 文档 6 个操作及静态资源路由、审计、同步、健康检查、日志下载、SQLite 回滚和制品安全 | `packages/daedalus-agent-management-web/server/test/app-smoke.test.js`、`packages/daedalus-agent-management-web/server/test/topic-ping-service.test.js`、`packages/daedalus-agent-management-web/server/test/agent-service-alignment.test.js`、`packages/daedalus-agent-management-web/server/test/env-config.test.js`、`packages/daedalus-agent-management-web/server/test/mcp-service.test.js`、`packages/daedalus-agent-management-web/server/management-test/mcp-sync.test.ts`、其他 `server/test/`、`server/management-test/` | | 管理 Web 前端 | API 客户端鉴权/超时/错误归一化、同步门控、重启轮询、格式化、浏览器覆盖契约、员工与模板列表操作精简、内置模板编辑、名称/编辑详情入口、详情 tabs 保存流程、员工概览主模型/备选模型/每日配额快捷修改及打开状态下的 Token 用量同步刷新、员工技能/业务系统/MCP 与模板设置布局一致及内置 Skill 隐藏、编辑页不展示同步调用 URL、模板文本文件切换后的编辑器可见性、设置页 API 文档入口 | `packages/daedalus-agent-management-web/src/**/*.test.js`、`packages/daedalus-agent-management-web/src/pages/AgentsPage.test.jsx`、`test/browser-coverage.test.js`、`test/run-browser-e2e.js`、`test/integration/website-full.js` | | invoke CLI | 参数、JSON/body/query 来源、Binding schema、socket 协议、超时、大小限制、Provider 错误和调用日志 | `packages/daedalus-agent-apps-invoke-cli/test/` | | MQTT 插件 | 账号配置、旧配置兼容、环境变量覆盖、连接/断线重连、通配符、入站路由、普通消息串行队列、停止与 steer/tell 控制队列、active-run 非阻塞 steer、运行时接收 Promise 未完成时的确认与原 final 投递、原始会话目标、OpenClaw deliver 的 text/kind 无加工透传、出站 envelope、日志 | `extensions/openclaw-plugins/daedalus-agent-mqtt-channel/src/*test.ts` | @@ -176,7 +176,7 @@ ## 8. 当前边界与使用前提 - [!] 管理控制台是管理员/运维面板，不是终端对话应用；终端用户通过 CUI 或 MQTT 接入 Agent。 -- [!] 同步 Agent HTTP 接口是服务间调用接口，必须使用服务 Bearer Token（本机请求按现有服务鉴权策略处理）；它只等待并返回最终结果，调用方需要自行处理 MQTT/Agent 超时或服务不可用错误。 +- [!] 同步 Agent HTTP 接口是服务间调用接口，必须使用服务 Bearer Token（本机请求按现有服务鉴权策略处理）；它只等待并返回最终结果，成功后刷新 Agent 用量以计入每日配额，调用方仍需要自行处理 MQTT/Agent 超时或服务不可用错误。 - [!] 业务系统当前按 HZG/活字格模型接入，Ontology ZIP 和 `index.md` 是必要前提。 - [!] Agent 只能访问配置白名单内的内网地址；白名单更新后必须重启容器。 - [!] 内置模板、内置 Skill、内置 Ontology 受删除保护；内置 Skill 和内置 Ontology 不能按普通资源直接编辑。 diff --git a/packages/daedalus-agent-management-web/README.md b/packages/daedalus-agent-management-web/README.md index d3bcdd6..0fcc8d0 100644 --- a/packages/daedalus-agent-management-web/README.md +++ b/packages/daedalus-agent-management-web/README.md @@ -103,7 +103,7 @@ npm run test:full - `GET /api/external/agents`（未传 `userName`/`id` 时仅返回“所有人”可用的 Agent） - `GET /api/external/agents?userName=zhangsan`（也支持使用 `id` 作为身份参数） -- `POST /api/external/agents/:agentId/call`（请求体为 `sessionId`、`senderId`、`text`；固定只返回 `kind=final` 最终结果） +- `POST /api/external/agents/:agentId/call`（请求体为 `sessionId`、`senderId`、`text`；固定只返回 `kind=final` 最终结果，并在成功后刷新 Agent 用量计入每日配额） - `GET /api/external/session?userName=zhangsan&agentId=finance-agent` - `GET /api/external/context?sessionId=...` diff --git a/packages/daedalus-agent-management-web/docs/spec.md b/packages/daedalus-agent-management-web/docs/spec.md index c04abd9..5746192 100644 --- a/packages/daedalus-agent-management-web/docs/spec.md +++ b/packages/daedalus-agent-management-web/docs/spec.md @@ -33,7 +33,7 @@ Token 用量由 `agent.usage.list` 同步到 `agents.usage_snapshot_json`。超 ### Agent 同步调用 -`POST /api/external/agents/{agentId}/call` 为指定 Agent 提供同步文本调用，JSON 请求体固定包含 `sessionId`、`senderId` 和 `text`。控制台将请求转发到 Agent MQTT 入站 Topic，并设置 `disableBlockStreaming: true`；只等待匹配 `sessionId` 的 `kind: "final"` 消息，忽略中间 `block` 消息后返回最终结果。接口使用服务 Bearer Token 鉴权，本机请求遵循现有服务鉴权例外。 +`POST /api/external/agents/{agentId}/call` 为指定 Agent 提供同步文本调用，JSON 请求体固定包含 `sessionId`、`senderId` 和 `text`。控制台将请求转发到 Agent MQTT 入站 Topic，并设置 `disableBlockStreaming: true`；只等待匹配 `sessionId` 的 `kind: "final"` 消息，忽略中间 `block` 消息后返回最终结果。成功取得最终结果后，控制台刷新 Agent 用量快照，使本次调用产生的 Token 计入每日配额并更新超额状态。接口使用服务 Bearer Token 鉴权，本机请求遵循现有服务鉴权例外。 ### 模板 diff --git a/packages/daedalus-agent-management-web/server/src/api/routes/index.js b/packages/daedalus-agent-management-web/server/src/api/routes/index.js index f18a991..58206c3 100644 --- a/packages/daedalus-agent-management-web/server/src/api/routes/index.js +++ b/packages/daedalus-agent-management-web/server/src/api/routes/index.js @@ -7,6 +7,9 @@ import multer from "multer"; import { createAuthMiddleware } from "../middleware/auth.js"; import { createInternalAuthMiddleware, createServiceApiAuthMiddleware } from "../middleware/internal.js"; import { badRequest, notFound, serviceUnavailable } from "../../utils/errors.js"; +import { createLogger, errorDetails } from "../../utils/logger.js"; + +const logger = createLogger("management-web-routes"); const upload = multer({ storage: multer.memoryStorage(), @@ -46,6 +49,27 @@ function normalizeTimeoutMs(value, fallback = 120000, max = 120000) { return Math.max(1000, Math.min(max, Math.floor(parsed))); } +async function refreshUsageAfterExternalAgentCall(services, agentId) { + const refreshAgentUsage = services.catalogSyncService?.refreshAgentUsage; + if (typeof refreshAgentUsage !== "function") { + return; + } + + try { + const result = await refreshAgentUsage.call(services.catalogSyncService, { + trigger: "external-agent-call" + }); + if (result?.status === "failed") { + logger.warn({ + agentId, + error: result.error_message || "unknown usage refresh error" + }, "External agent call usage refresh failed"); + } + } catch (error) { + logger.warn({ agentId, error: errorDetails(error) }, "External agent call usage refresh failed"); + } +} + function normalizeSystemLogType(value) { const type = String(value || "").trim(); if (!SYSTEM_LOG_TYPE_VALUES.has(type)) { @@ -357,6 +381,7 @@ export function createRoutes(services) { senderId: payload.senderId, text: payload.text }); + await refreshUsageAfterExternalAgentCall(services, req.params.agentId); res.json(result); })); diff --git a/packages/daedalus-agent-management-web/server/src/docs/openapi.js b/packages/daedalus-agent-management-web/server/src/docs/openapi.js index fcf3d43..05e328d 100644 --- a/packages/daedalus-agent-management-web/server/src/docs/openapi.js +++ b/packages/daedalus-agent-management-web/server/src/docs/openapi.js @@ -234,7 +234,7 @@ const ROUTES = [ ], successSchema: { type: "object", properties: { items: { type: "array", items: { $ref: "#/components/schemas/ExternalAgentDirectoryItem" } } } } }), - route("post", "/external/agents/:agentId/call", "外部服务", "同步调用 Agent", "向指定数字员工发送一条文本消息并等待最终结果。服务端固定关闭 block 流式返回，只返回匹配请求会话的 `kind=final` 消息。", { + route("post", "/external/agents/:agentId/call", "外部服务", "同步调用 Agent", "向指定数字员工发送一条文本消息并等待最终结果。服务端固定关闭 block 流式返回，只返回匹配请求会话的 `kind=final` 消息，并在成功后刷新 Agent 用量以计入每日配额。", { authDescription: "需要服务 API Bearer Token。", body: syncAgentMessageBody, successSchema: { $ref: "#/components/schemas/ExternalAgentSyncResponse" }, diff --git a/packages/daedalus-agent-management-web/server/test/app-smoke.test.js b/packages/daedalus-agent-management-web/server/test/app-smoke.test.js index 5f907f0..5b11045 100644 --- a/packages/daedalus-agent-management-web/server/test/app-smoke.test.js +++ b/packages/daedalus-agent-management-web/server/test/app-smoke.test.js @@ -116,6 +116,8 @@ it("supports anonymous public external agent directories and id identity aliases it("handles synchronous agent requests and returns only the final result", async () => { const { app, services } = createServerApp(); const calls = []; + const usageRefreshes = []; + let usageRefreshResult = { status: "success" }; services.topicPingService.syncAgentMessage = async (agentId, payload) => { calls.push({ agentId, payload }); return { @@ -124,6 +126,13 @@ it("handles synchronous agent requests and returns only the final result", async text: "同步结果" }; }; + services.catalogSyncService.refreshAgentUsage = async (options) => { + usageRefreshes.push(options); + if (usageRefreshResult === "throw") { + throw new Error("usage refresh unavailable"); + } + return usageRefreshResult; + }; const server = await listen(app); const baseUrl = `http://127.0.0.1:${server.address().port}`; @@ -144,6 +153,7 @@ it("handles synchronous agent requests and returns only the final result", async agentId: "finance-agent", payload: { sessionId: "session-1", senderId: "sender-1", text: "hello" } }]); + expect(usageRefreshes).toEqual([{ trigger: "external-agent-call" }]); const invalidResponse = await fetch(`${baseUrl}/api/external/agents/finance-agent/call`, { method: "POST", @@ -151,6 +161,34 @@ it("handles synchronous agent requests and returns only the final result", async body: JSON.stringify({ sessionId: "session-1", senderId: "sender-1" }) }); expect(invalidResponse.status).toBe(400); + expect(usageRefreshes).toHaveSize(1); + + usageRefreshResult = { status: "failed", error_message: "usage source unavailable" }; + const failedRefreshResponse = await fetch(`${baseUrl}/api/external/agents/finance-agent/call`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ sessionId: "session-2", senderId: "sender-1", text: "hello again" }) + }); + expect(failedRefreshResponse.status).toBe(200); + expect(await failedRefreshResponse.json()).toEqual({ + sessionId: "session-2", + kind: "final", + text: "同步结果" + }); + + usageRefreshResult = "throw"; + const thrownRefreshResponse = await fetch(`${baseUrl}/api/external/agents/finance-agent/call`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ sessionId: "session-3", senderId: "sender-1", text: "hello once more" }) + }); + expect(thrownRefreshResponse.status).toBe(200); + expect(await thrownRefreshResponse.json()).toEqual({ + sessionId: "session-3", + kind: "final", + text: "同步结果" + }); + expect(usageRefreshes).toHaveSize(3); } finally { await close(server); } -- Gitee From decb08469aaea342b3d34898d49e4c78620bfefd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=AE=81=E4=BC=9F?= Date: Tue, 1 Sep 2026 11:28:52 +0800 Subject: [PATCH 3/9] =?UTF-8?q?=E8=B0=83=E6=95=B4=E5=90=8C=E6=AD=A5?= =?UTF-8?q?=E8=B0=83=E7=94=A8=E6=8E=A5=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- doc/system-feature-list.md | 26 +++---- docker/assets/scripts/kernel-startup.sh | 10 +++ .../daedalus-agent-management-web/README.md | 10 +-- .../docs/spec.md | 2 +- .../server/src/api/routes/index.js | 21 ++++-- .../server/src/docs/openapi.js | 22 +++--- .../server/src/services/agent-service.js | 11 ++- .../server/src/services/external-service.js | 53 ++++++++++++-- .../services/runtime-env-config-service.js | 37 ++++++++++ .../server/src/services/topic-ping-service.js | 18 ++--- .../server/test/agent-user-policy.test.js | 7 ++ .../server/test/app-smoke.test.js | 20 ++--- .../server/test/external-service.test.js | 55 +++++++++++++- .../test/runtime-env-config-service.test.js | 49 +++++++++++++ .../test/runtime-settings-route.test.js | 62 ++++++++++++++++ .../server/test/topic-ping-service.test.js | 21 +++--- .../src/app/App.jsx | 11 +++ .../src/pages/AgentsPage.jsx | 6 +- .../src/pages/SettingsPage.jsx | 73 ++++++++++++++++--- .../pages/agents/AgentWorkspaceWorkbench.jsx | 2 +- .../test/integration/website-full.js | 5 ++ .../test/run-browser-e2e.js | 6 ++ .../upgrade/openclaw-upgrade-contract.test.js | 16 +++- 23 files changed, 449 insertions(+), 94 deletions(-) diff --git a/doc/system-feature-list.md b/doc/system-feature-list.md index 5b624ff..2bd108c 100644 --- a/doc/system-feature-list.md +++ b/doc/system-feature-list.md @@ -10,7 +10,7 @@ - [x][T] 默认开放管理端口 `3030`、预留服务端口 `3031-3050`、MQTT `1883`、MinIO S3 API `9000`。 - [x] 使用命名 Docker volume 持久化 `/var/daedalus-agent`，保存 OpenClaw 状态、工作区、模板、Ontology、SQLite 数据、制品、日志、MQTT/MinIO 数据和运行状态。 - [x][T] 启动时校验 `env.json`、`config.json`；缺失时写入示例文件并要求用户检查后重启；拒绝用户覆盖由系统托管的内部路径、MQTT、MinIO、Topic 和 IPC 配置。 -- [x][T] 自动生成并持久化 MQTT/MinIO 运行凭据；管理控制台可显示运行凭据。 +- [x][T] 自动生成并持久化 MQTT/MinIO 运行凭据；管理控制台可显示运行凭据，并支持管理员确认后重置两组密码和请求容器重启；容器启动时会将持久化 OpenClaw MQTT 账号密码同步为当前运行凭据。 - [x] 以 `daedalus-agent` 运行 OpenClaw/Agent，以 `daedalus-svc` 运行守护进程、Broker、MinIO、管理端和 invoke service，通过共享组控制只读目录、日志和 socket 访问。 - [x][T] 对 Agent 进程实施内网访问白名单和 iptables 限制；管理控制台可维护白名单，IP 地址变更在容器重启后生效。 - [x][T] 提供启动步骤耗时日志、子服务守护、服务重启、容器重启请求文件和快速重启保护。 @@ -73,19 +73,19 @@ ### 2.6 数字员工（Agent）管理 - [x][T] Agent 新增、列表、详情、编辑、启用/停用、删除；ID/slug 校验并保持与远端 Agent 一致。 -- [x][T] 配置名称、描述、标签、模板、主/备选 LLM、每日 Token 限额、状态和授权用户；概览 tab 在编辑状态下可分别修改主模型、备选模型和每日配额；用户范围支持指定用户或所有人，所有人模式对当前及未来新增的 Agent 平台用户动态生效。 -- [x][T] 数字员工名称和编辑按钮均从列表进入全屏详情，操作列仅保留编辑、删除；详情包含只读概览、文件（默认加载 `AGENTS.md`）、技能、业务系统、MCP 和分配 tabs；技能、业务系统、MCP 三个 tab 复用员工模板对应的设置布局（模板式 `List`、`Segmented`、搜索、勾选、状态摘要和保存区域），数字员工技能页仅展示当前 Agent 专属 Skill，不展示全局或内置 Skill，分配复合框占满区块宽度，支持从现有用户选择或直接添加用户名，并支持全部当前及未来用户。 +- [x][T] 配置名称、描述、标签、模板、主/备选 LLM、每日 Token 限额、状态和授权用户；概览 tab 在编辑状态下可分别修改主模型、备选模型和每日配额；用户范围支持指定用户或全部用户，全部用户模式不检查用户名是否存在于 Agent 平台用户列表，任何用户名均可使用。 +- [x][T] 数字员工名称和编辑按钮均从列表进入全屏详情，操作列仅保留编辑、删除；详情包含只读概览、文件（默认加载 `AGENTS.md`）、技能、业务系统、MCP 和分配 tabs；技能、业务系统、MCP 三个 tab 复用员工模板对应的设置布局（模板式 `List`、`Segmented`、搜索、勾选、状态摘要和保存区域），数字员工技能页仅展示当前 Agent 专属 Skill，不展示全局或内置 Skill，分配复合框占满区块宽度，支持从现有用户选择或直接添加用户名；全部用户模式可使用任意用户名，指定用户模式按列表中的显式授权生效。 - [x][T] 状态包括 `normal`、`disabled`、`overlimit`；用量刷新完成后，Agent 列表与已打开的详情概览同步展示最新 Token 用量；超过每日限额时自动标记 `overlimit`，不可直接恢复为 `normal` 直到超额问题解决。 - [x][T] 创建/更新具备远端先行、本地 SQLite 落库、失败补偿和回滚；远端已不存在的 Agent 可清理本地记录。 - [x][T] Agent 用户目录支持分页搜索、单个新增、换行批量导入、删除和用量/关联员工查询。 -- [x][T] 为 Agent 分配或移除用户，支持带策略版本的“所有人/指定用户”访问范围；外部目录按 `userName`/`id` 返回用户有权限的 Agent，未传身份时仅返回“所有人”可用的 Agent，并在每个目录项返回 `is_public` 标记，所有人模式对新增用户即时生效。 -- [x][T] 服务层维护 Agent 用户访问策略 `selected/all`、策略版本和乐观并发更新；数据库 v17 迁移将历史 Agent 初始化为 `selected`，保持原有显式授权语义，管理端提供“所有人/指定用户”配置入口。 +- [x][T] 为 Agent 分配或移除用户，支持带策略版本的“全部用户/指定用户”访问范围；全部用户模式不检查 `userName` 是否存在于 `agent_users`，外部目录和外部会话均允许任何用户名使用，首次创建会话时自动登记该用户名；指定用户模式仍要求显式授权；未传身份时仅返回全部用户可用的 Agent，并在每个目录项返回 `is_public` 标记。 +- [x][T] 服务层维护 Agent 用户访问策略 `selected/all`、策略版本和乐观并发更新；数据库 v17 迁移将历史 Agent 初始化为 `selected`，保持原有显式授权语义，管理端提供“全部用户/指定用户”配置入口。 - [x][T] 配置 Agent 业务系统权限（全部/选定）和 MCP 权限（全部/选定），带策略版本号并进行乐观并发控制；MCP 无权限服务同步为普通工具和沙箱工具策略中的 Agent deny 规则，避免通过沙箱会话绕过隔离。 - [x][T] Agent 工作区导出为 ZIP；导出制品写入本地临时制品存储，下载后清理，不依赖 S3。 - [x][T] Agent 工作区 Markdown 文件列表、读取、创建、保存、移动/重命名、删除、下载，使用 etag 做乐观并发校验；文件页不展示额外的“新会话生效”提示。 - [x][T] 恢复 Agent 默认 `AGENTS.md`；Agent Skill 列表、ZIP 安装/替换、删除、导出，支持 etag 和 slug 校验。 - [x][T] 管理控制台提供 Agent 入站 Topic 探测、管理 Topic 探测和对话测试；对话测试展示发送消息、回复消息、步骤耗时和完整 MQTT 报文。 -- [x][T] 为每个 Agent 暴露同步 HTTP 调用 URL；`POST /api/external/agents/{agentId}/call` 接收 `sessionId`、`senderId`、`text`，通过 MQTT 请求 Agent 并固定只返回匹配会话的 `kind=final` 最终结果（发送 `disableBlockStreaming: true`，忽略中间 `block`）；成功后刷新 Agent 用量快照，将本次调用产生的 Token 计入每日配额并更新超额状态。 +- [x][T] 为每个 Agent 暴露同步 HTTP 调用 URL；`POST /api/external/agents/{agentId}/call` 接收 `senderId`、`text`，服务端为每次请求生成一次性 MQTT 关联会话 ID，通过 MQTT 请求 Agent 并固定只返回匹配临时会话的 `kind=final` 最终结果（发送 `disableBlockStreaming: true`，忽略中间 `block`）；临时会话使用后废弃，不写入 `external_sessions`/`external_session_cookies`；成功后刷新 Agent 用量快照，将本次调用产生的 Token 计入每日配额并更新超额状态。 ### 2.7 MCP 服务管理 @@ -110,16 +110,16 @@ - [x][T] 系统日志页包含审计日志、业务系统调用日志和内核/管理动作调用日志，并支持详情查看。 - [x][T] 系统日志下载支持按日期选择 MQTT、MinIO、内核、应用、OpenClaw、管理端、invoke service、MQTT 通讯和交互日志。 -- [x][T] 基础设置维护门户信息、对话/业务应用服务器地址、运行凭据和 IP 白名单。 -- [x][T] 设置页展示 MQTT/MinIO/服务运行健康状态，支持刷新健康缓存。 +- [x][T] 基础设置维护门户信息、对话/业务应用服务器地址、运行凭据和 IP 白名单；运行凭据支持重置 MQTT/MinIO 密码，重置会请求容器重启以使新密码生效。 +- [x][T] 设置页展示 MQTT/MinIO/服务运行健康状态，支持刷新健康缓存；运行凭据区分“重置密码”和“刷新状态”操作，并在重置前要求确认。 - [x][T] 支持目录同步、查询 Gateway 状态、重启 Gateway、重启容器、执行任意已注册管理指令和运行服务器诊断。 - [T] 设置页对重启进行轮询等待、提示服务重启状态；后台实时更新通过 WebSocket/update notifier 推送。 ## 3. 外部与内部 API -- [x][T] 外部服务 API：按 `userName`/`id` 查询可用 Agent；未传身份时仅查询“所有人”可用的 Agent，并返回每项的 `is_public` 标记；支持创建/复用用户-Agent 会话、获取会话上下文、获取业务系统 Cookie、记录调用日志；支持 `POST /api/external/agents/{agentId}/call` 同步发送文本并只返回 `final` 最终结果，成功后刷新用量并计入每日配额。 +- [x][T] 外部服务 API：按 `userName`/`id` 查询可用 Agent；全部用户模式不检查用户名是否存在，未传身份时仅查询“全部用户”可用的 Agent，并返回每项的 `is_public` 标记；支持创建/复用用户-Agent 会话（目录外用户名首次创建会话时自动登记）、获取会话上下文、获取业务系统 Cookie、记录调用日志；支持 `POST /api/external/agents/{agentId}/call` 同步发送文本，由服务端创建并销毁一次性 MQTT 关联会话且不写入外部会话表，只返回 `final` 最终结果，成功后刷新用量并计入每日配额。 - [x][T] 管理 Web 提供面向 `/api/external/*` 对外接口的 OpenAPI 3.0 文档和本地 Swagger UI：文档入口为 `/doc`，规范 JSON 为 `/doc/openapi.json`，包含应用场景、鉴权、参数用途/限制/可选项、请求体和常见错误说明；基础设置的版本信息下方提供“API 文档”入口。 -- [x][T] 外部会话 ID 使用 `s-` 加 30 位数字格式；同一用户-Agent 复用会话；会话创建校验 Agent 状态和用户授权。 +- [x][T] 持久外部会话 ID 使用 `s-` 加 30 位数字格式；同一用户-Agent 复用会话；会话创建校验 Agent 状态和用户授权。同步调用使用独立的一次性 MQTT 关联 ID，不属于持久外部会话。 - [x][T] 外部调用在每次解析业务系统时重新校验会话有效性、Agent 状态、业务系统状态和 Agent 业务系统策略。 - [x][T] 内部 CUI API：查询可用 Agent 目录；内部 invoke API：解析授权业务系统、获取业务系统 base URL、记录调用日志。 - [x][T] API 鉴权分层：管理端登录 Token、服务 Bearer Token、内部请求校验；localhost 请求按配置可走内部通道。 @@ -166,12 +166,12 @@ | 范围 | 主要验证内容 | 代表测试位置 | | --- | --- | --- | -| 管理 Web 服务 | 路由、鉴权、用户/Agent/模板/Skill/LLM/MCP/业务系统 CRUD、内置模板编辑/重命名与删除保护、模板主/备选模型和 Agent 继承、Agent 用户访问策略、Agent 多实例 MCP 选定服务同步及普通/沙箱 deny 投影、外部 Agent 目录匿名公开筛选与 `is_public` 标记、同步 Agent HTTP 接口（参数校验、服务鉴权、MQTT final-only 行为、成功后刷新用量并计入每日配额）、仅面向 `/api/external/*` 的 OpenAPI/Swagger 文档 6 个操作及静态资源路由、审计、同步、健康检查、日志下载、SQLite 回滚和制品安全 | `packages/daedalus-agent-management-web/server/test/app-smoke.test.js`、`packages/daedalus-agent-management-web/server/test/topic-ping-service.test.js`、`packages/daedalus-agent-management-web/server/test/agent-service-alignment.test.js`、`packages/daedalus-agent-management-web/server/test/env-config.test.js`、`packages/daedalus-agent-management-web/server/test/mcp-service.test.js`、`packages/daedalus-agent-management-web/server/management-test/mcp-sync.test.ts`、其他 `server/test/`、`server/management-test/` | -| 管理 Web 前端 | API 客户端鉴权/超时/错误归一化、同步门控、重启轮询、格式化、浏览器覆盖契约、员工与模板列表操作精简、内置模板编辑、名称/编辑详情入口、详情 tabs 保存流程、员工概览主模型/备选模型/每日配额快捷修改及打开状态下的 Token 用量同步刷新、员工技能/业务系统/MCP 与模板设置布局一致及内置 Skill 隐藏、编辑页不展示同步调用 URL、模板文本文件切换后的编辑器可见性、设置页 API 文档入口 | `packages/daedalus-agent-management-web/src/**/*.test.js`、`packages/daedalus-agent-management-web/src/pages/AgentsPage.test.jsx`、`test/browser-coverage.test.js`、`test/run-browser-e2e.js`、`test/integration/website-full.js` | +| 管理 Web 服务 | 路由、鉴权、用户/Agent/模板/Skill/LLM/MCP/业务系统 CRUD、内置模板编辑/重命名与删除保护、模板主/备选模型和 Agent 继承、Agent 用户访问策略（全部用户模式允许目录外未知用户名、指定用户模式仍需显式授权）、Agent 多实例 MCP 选定服务同步及普通/沙箱 deny 投影、外部 Agent 目录匿名公开筛选与 `is_public` 标记、同步 Agent HTTP 接口（参数校验、服务鉴权、MQTT final-only 行为、成功后刷新用量并计入每日配额）、运行凭据密码重置及容器重启请求、仅面向 `/api/external/*` 的 OpenAPI/Swagger 文档 6 个操作及静态资源路由、审计、同步、健康检查、日志下载、SQLite 回滚和制品安全 | `packages/daedalus-agent-management-web/server/test/app-smoke.test.js`、`packages/daedalus-agent-management-web/server/test/runtime-env-config-service.test.js`、`packages/daedalus-agent-management-web/server/test/runtime-settings-route.test.js`、`packages/daedalus-agent-management-web/server/test/topic-ping-service.test.js`、`packages/daedalus-agent-management-web/server/test/agent-user-policy.test.js`、`packages/daedalus-agent-management-web/server/test/external-service.test.js`、`packages/daedalus-agent-management-web/server/test/agent-service-alignment.test.js`、`packages/daedalus-agent-management-web/server/test/env-config.test.js`、`packages/daedalus-agent-management-web/server/test/mcp-service.test.js`、`packages/daedalus-agent-management-web/server/management-test/mcp-sync.test.ts`、其他 `server/test/`、`server/management-test/` | +| 管理 Web 前端 | API 客户端鉴权/超时/错误归一化、同步门控、重启轮询、格式化、浏览器覆盖契约、员工与模板列表操作精简、内置模板编辑、名称/编辑详情入口、详情 tabs 保存流程、员工概览主模型/备选模型/每日配额快捷修改及打开状态下的 Token 用量同步刷新、员工技能/业务系统/MCP 与模板设置布局一致及内置 Skill 隐藏、编辑页不展示同步调用 URL、模板文本文件切换后的编辑器可见性、设置页 API 文档入口、运行凭据重置按钮/确认框及“刷新状态”按钮 | `packages/daedalus-agent-management-web/src/**/*.test.js`、`packages/daedalus-agent-management-web/src/pages/AgentsPage.test.jsx`、`packages/daedalus-agent-management-web/test/browser-coverage.test.js`、`packages/daedalus-agent-management-web/test/run-browser-e2e.js`、`packages/daedalus-agent-management-web/test/integration/website-full.js` | | invoke CLI | 参数、JSON/body/query 来源、Binding schema、socket 协议、超时、大小限制、Provider 错误和调用日志 | `packages/daedalus-agent-apps-invoke-cli/test/` | | MQTT 插件 | 账号配置、旧配置兼容、环境变量覆盖、连接/断线重连、通配符、入站路由、普通消息串行队列、停止与 steer/tell 控制队列、active-run 非阻塞 steer、运行时接收 Promise 未完成时的确认与原 final 投递、原始会话目标、OpenClaw deliver 的 text/kind 无加工透传、出站 envelope、日志 | `extensions/openclaw-plugins/daedalus-agent-mqtt-channel/src/*test.ts` | | Docker 集成 | 登录、版本/同步、Provider/Model/模板/MCP/Agent 生命周期、模板继承、工作区/Skill、Ontology、MQTT `/stop`/`/steer`/`/tell` 的 final 确认及普通队列保留、重启、启动钩子、包持久化、文件输出、会话隔离和压测 | `test/case/integration/`、`test/suite/integration/`、`test/case/stress/` | -| Docker 升级与品牌契约 | OpenClaw/Node 版本、插件 seed、迁移、重启保护、DA_ 环境变量前缀和视觉品牌 | `test/case/upgrade/`、`test/case/branding/` | +| Docker 升级与品牌契约 | OpenClaw/Node 版本、插件 seed、迁移、重启保护、持久化 MQTT 账号密码与运行凭据同步、DA_ 环境变量前缀和视觉品牌 | `test/case/upgrade/openclaw-upgrade-contract.test.js`、`test/case/upgrade/`、`test/case/branding/` | ## 8. 当前边界与使用前提 diff --git a/docker/assets/scripts/kernel-startup.sh b/docker/assets/scripts/kernel-startup.sh index 2cd48c6..b788a59 100644 --- a/docker/assets/scripts/kernel-startup.sh +++ b/docker/assets/scripts/kernel-startup.sh @@ -2217,6 +2217,7 @@ patch_openclaw_config() { local tmp_json local gateway_auth_token="${DA_OPENCLAW_GATEWAY_AUTH_TOKEN:-daedalus-agent-local-gateway-token}" local gateway_max_concurrent + local mqtt_password="${DA_MQTT_CHANNEL_PASSWORD:-}" mkdir -p "$(dirname "$config_path")" gateway_max_concurrent="$(normalize_openclaw_gateway_max_concurrent)" @@ -2237,6 +2238,7 @@ patch_openclaw_config() { --arg workspace "$DA_OPENCLAW_WORKSPACE_DIR" \ --arg qmdCommand "$QMD_COMMAND" \ --arg gatewayAuthToken "$gateway_auth_token" \ + --arg mqttPassword "$mqtt_password" \ --arg seedMqttPluginPath "$SEED_MQTT_PLUGIN_PATH" \ --argjson gatewayMaxConcurrent "$gateway_max_concurrent" \ ' @@ -2272,6 +2274,14 @@ patch_openclaw_config() { | .plugins.allow = ((.plugins.allow // []) + ["daedalus-agent-mqtt-channel"] | unique) | .plugins.load.paths = ((.plugins.load.paths // []) + [$seedMqttPluginPath] | unique) | .plugins.entries["daedalus-agent-mqtt-channel"].enabled = true + | if ((.channels["daedalus-agent-mqtt-channel"].accounts? | type) == "object") and ($mqttPassword != "") then + .channels["daedalus-agent-mqtt-channel"].accounts |= with_entries( + .value.username = (.value.username // "daedalus-agent") + | .value.password = $mqttPassword + ) + else + . + end ' "$config_source" > "$tmp_json"; then rm -f "$tmp_json" return 1 diff --git a/packages/daedalus-agent-management-web/README.md b/packages/daedalus-agent-management-web/README.md index 0fcc8d0..fe45bb6 100644 --- a/packages/daedalus-agent-management-web/README.md +++ b/packages/daedalus-agent-management-web/README.md @@ -101,13 +101,13 @@ npm run test:full 面向 CUI： -- `GET /api/external/agents`（未传 `userName`/`id` 时仅返回“所有人”可用的 Agent） -- `GET /api/external/agents?userName=zhangsan`（也支持使用 `id` 作为身份参数） -- `POST /api/external/agents/:agentId/call`（请求体为 `sessionId`、`senderId`、`text`；固定只返回 `kind=final` 最终结果，并在成功后刷新 Agent 用量计入每日配额） -- `GET /api/external/session?userName=zhangsan&agentId=finance-agent` +- `GET /api/external/agents`（未传 `userName`/`id` 时仅返回“全部用户”可用的 Agent） +- `GET /api/external/agents?userName=zhangsan`（全部用户模式不检查用户名是否存在；指定用户模式按显式授权筛选，也支持使用 `id` 作为身份参数） +- `POST /api/external/agents/:agentId/call`（请求体为 `senderId`、`text`；服务端每次生成一次性 MQTT 关联会话，固定只返回 `kind=final` 最终结果，临时会话不写入 `external_sessions`，并在成功后刷新 Agent 用量计入每日配额） +- `GET /api/external/session?userName=zhangsan&agentId=finance-agent`（Agent 为“全部用户”时不要求用户名已在员工列表中） - `GET /api/external/context?sessionId=...` -Agent 目录项提供该同步接口的 URL。调用需要服务 Bearer Token；服务端通过 Agent MQTT 入站/出站 Topic 转发请求，并固定发送 `disableBlockStreaming: true`，忽略中间 `block` 消息。 +Agent 目录项提供该同步接口的 URL。调用需要服务 Bearer Token；请求只需提供 `senderId` 和 `text`，服务端为每次调用生成一次性 MQTT 关联会话，使用后立即废弃且不在 `external_sessions` 中留存；服务端通过 Agent MQTT 入站/出站 Topic 转发请求，并固定发送 `disableBlockStreaming: true`，忽略中间 `block` 消息。 API 文档： diff --git a/packages/daedalus-agent-management-web/docs/spec.md b/packages/daedalus-agent-management-web/docs/spec.md index 5746192..6803069 100644 --- a/packages/daedalus-agent-management-web/docs/spec.md +++ b/packages/daedalus-agent-management-web/docs/spec.md @@ -33,7 +33,7 @@ Token 用量由 `agent.usage.list` 同步到 `agents.usage_snapshot_json`。超 ### Agent 同步调用 -`POST /api/external/agents/{agentId}/call` 为指定 Agent 提供同步文本调用，JSON 请求体固定包含 `sessionId`、`senderId` 和 `text`。控制台将请求转发到 Agent MQTT 入站 Topic，并设置 `disableBlockStreaming: true`；只等待匹配 `sessionId` 的 `kind: "final"` 消息，忽略中间 `block` 消息后返回最终结果。成功取得最终结果后，控制台刷新 Agent 用量快照，使本次调用产生的 Token 计入每日配额并更新超额状态。接口使用服务 Bearer Token 鉴权，本机请求遵循现有服务鉴权例外。 +`POST /api/external/agents/{agentId}/call` 为指定 Agent 提供同步文本调用，JSON 请求体固定包含 `senderId` 和 `text`，不要求调用方传入 `sessionId`。控制台为每次请求生成一次性 MQTT 关联会话 ID，仅在等待本次回复期间使用，结束后立即废弃且不会写入 `external_sessions` 或 `external_session_cookies`。控制台将请求转发到 Agent MQTT 入站 Topic，并设置 `disableBlockStreaming: true`；只等待匹配该临时 ID 的 `kind: "final"` 消息，忽略中间 `block` 消息后返回最终结果。成功取得最终结果后，控制台刷新 Agent 用量快照，使本次调用产生的 Token 计入每日配额并更新超额状态。接口使用服务 Bearer Token 鉴权，本机请求遵循现有服务鉴权例外。 ### 模板 diff --git a/packages/daedalus-agent-management-web/server/src/api/routes/index.js b/packages/daedalus-agent-management-web/server/src/api/routes/index.js index 58206c3..b4e4da3 100644 --- a/packages/daedalus-agent-management-web/server/src/api/routes/index.js +++ b/packages/daedalus-agent-management-web/server/src/api/routes/index.js @@ -367,9 +367,6 @@ export function createRoutes(services) { router.post("/external/agents/:agentId/call", requireServiceApiAuth, asyncRoute(async (req, res) => { const payload = req.body || {}; - if (!String(payload.sessionId || "").trim()) { - throw badRequest("sessionId 不能为空"); - } if (!String(payload.senderId || "").trim()) { throw badRequest("senderId 不能为空"); } @@ -377,7 +374,6 @@ export function createRoutes(services) { throw badRequest("文本不能为空"); } const result = await services.topicPingService.syncAgentMessage(req.params.agentId, { - sessionId: payload.sessionId, senderId: payload.senderId, text: payload.text }); @@ -828,7 +824,7 @@ export function createRoutes(services) { req, "编辑数字员工用户权限", `数字员工 ${agent?.slug || req.params.id}；模式：${result.mode}；用户：${result.mode === "all" - ? "所有人" + ? "全部用户" : result.permission_usernames.join(", ") || "无"}` ); res.json(result); @@ -1319,6 +1315,21 @@ export function createRoutes(services) { res.json(await healthService.checkAll({ timeoutMs })); })); + router.post("/settings/runtime-credentials/reset-password", asyncRoute(async (req, res) => { + services.authService.assertAdmin(req.currentUser); + const credentials = requireRuntimeEnvConfigService(services).resetRuntimeCredentialPasswords(); + const restart = requireContainerService(services).requestRestart({ + requestedBy: req.currentUser?.username || "" + }); + writeAuditLog( + services, + req, + "重置运行凭据密码", + "重置 MQTT/MinIO 运行凭据密码并请求重启容器" + ); + res.json({ credentials, restart }); + })); + router.get("/settings/access-tokens", asyncRoute(async (_req, res) => { res.json(services.portalService.listAccessTokens()); })); diff --git a/packages/daedalus-agent-management-web/server/src/docs/openapi.js b/packages/daedalus-agent-management-web/server/src/docs/openapi.js index 05e328d..393d97d 100644 --- a/packages/daedalus-agent-management-web/server/src/docs/openapi.js +++ b/packages/daedalus-agent-management-web/server/src/docs/openapi.js @@ -149,11 +149,10 @@ const agentBody = jsonBody("数字员工配置。创建时需要 id/agent_name/t tags: { type: "array", items: { type: "string" }, description: "标签列表，可选。" }, restart: booleanField("是否在远端变更后重启运行时；默认按服务端策略执行。") }); -const syncAgentMessageBody = jsonBody("同步 Agent 请求。服务端通过 MQTT 等待最终回复，不向调用方返回中间 block。", { - sessionId: textField("本次请求的会话 ID；用于关联 MQTT 回复。", { minLength: 1 }), +const syncAgentMessageBody = jsonBody("同步 Agent 请求。服务端每次创建一次性 MQTT 关联会话，不向调用方返回中间 block，且不会将临时会话写入 external_sessions。", { senderId: textField("发送者 ID；用于 Agent 会话路由。", { minLength: 1 }), text: textField("发送给 Agent 的文本；不能为空。", { minLength: 1 }) -}, ["sessionId", "senderId", "text"]); +}, ["senderId", "text"]); const settingsBody = jsonBody("门户基础设置。", { portal_name: textField("门户名称，不能为空。"), brand_subtitle: textField("门户副标题，不能为空。"), @@ -226,21 +225,21 @@ const ROUTES = [ errorMessage: textField("失败原因，可选。") }) }), - route("get", "/external/agents", "外部服务", "查询外部 Agent 目录", "CUI 展示可用数字员工。传 `userName` 或 `id` 时返回该用户有权限的 Agent；两者都不传时仅返回用户范围为“所有人”的 Agent。每项包含 `is_public`。", { + route("get", "/external/agents", "外部服务", "查询外部 Agent 目录", "CUI 展示可用数字员工。传 `userName` 或 `id` 时，全部用户模式不检查用户名是否存在，指定用户模式按显式授权返回 Agent；两者都不传时仅返回用户范围为“全部用户”的 Agent。每项包含 `is_public`。", { authDescription: "需要服务 API Bearer Token。", query: [ - queryParam("userName", "用户身份，用于筛选指定用户可访问的 Agent；与 `id` 二选一。"), - queryParam("id", "用户身份别名；与 `userName` 二选一。") + queryParam("userName", "用户身份；全部用户模式不要求该用户名存在，指定用户模式按该用户名筛选；与 `id` 二选一。"), + queryParam("id", "用户身份别名；全部用户模式不要求该用户名存在，指定用户模式按该用户名筛选；与 `userName` 二选一。") ], successSchema: { type: "object", properties: { items: { type: "array", items: { $ref: "#/components/schemas/ExternalAgentDirectoryItem" } } } } }), - route("post", "/external/agents/:agentId/call", "外部服务", "同步调用 Agent", "向指定数字员工发送一条文本消息并等待最终结果。服务端固定关闭 block 流式返回，只返回匹配请求会话的 `kind=final` 消息，并在成功后刷新 Agent 用量以计入每日配额。", { + route("post", "/external/agents/:agentId/call", "外部服务", "同步调用 Agent", "向指定数字员工发送一条文本消息并等待最终结果。服务端每次生成一次性 MQTT 关联会话，固定关闭 block 流式返回，只返回匹配该临时会话的 `kind=final` 消息；临时会话仅用于本次请求，不写入 `external_sessions`，并在请求结束后废弃。成功后刷新 Agent 用量以计入每日配额。", { authDescription: "需要服务 API Bearer Token。", body: syncAgentMessageBody, successSchema: { $ref: "#/components/schemas/ExternalAgentSyncResponse" }, errors: { 503: "MQTT 连接、发送或等待最终回复失败。" } }), - route("get", "/external/session", "外部服务", "创建或复用外部会话", "按用户与 Agent 复用会话；创建时校验 Agent 状态、用户授权和必要 Cookie。", { + route("get", "/external/session", "外部服务", "创建或复用外部会话", "按用户与 Agent 复用会话；创建时校验 Agent 状态和用户授权，全部用户模式不要求用户名存在于 Agent 平台用户列表。", { authDescription: "需要服务 API Bearer Token。", query: [ queryParam("userName", "外部用户名称；不能为空。", {}, true), @@ -428,7 +427,7 @@ const ROUTES = [ route("get", "/agents", "数字员工", "查询数字员工", "返回 Agent 列表、状态、模型、限额、权限和远端同步摘要。"), route("get", "/agents/{id}/business-systems", "数字员工权限", "查询 Agent 业务系统策略", "读取 Agent 的全部/选定业务系统访问范围和策略版本。"), route("get", "/agents/{id}/user-policy", "数字员工权限", "查询 Agent 用户策略", "读取 Agent 的 all/selected 用户范围、策略版本及用户列表。"), - route("put", "/agents/{id}/user-policy", "数字员工权限", "设置 Agent 用户策略", "管理员配置“所有人”或“指定用户”访问范围；policy_version 用于乐观并发控制。", { + route("put", "/agents/{id}/user-policy", "数字员工权限", "设置 Agent 用户策略", "管理员配置“全部用户”或“指定用户”访问范围；全部用户模式不检查用户名是否存在，指定用户模式按显式用户名授权；policy_version 用于乐观并发控制。", { body: jsonBody("用户访问策略。mode 为 all 或 selected；selected 时提供 permission_usernames。", { mode: textField("用户访问模式。", { enum: ["all", "selected"] }), permission_usernames: { type: "array", items: { type: "string" }, description: "selected 模式下的用户名列表。" }, @@ -599,6 +598,7 @@ const ROUTES = [ queryParam("timeout_ms", "单项检查超时时间，范围 5000-30000 毫秒。", { type: "integer", minimum: 5000, maximum: 30000 }) ] }), + route("post", "/settings/runtime-credentials/reset-password", "系统设置", "重置运行凭据密码", "重新生成 MQTT/MinIO 运行凭据密码并请求重启容器，使新密码生效。"), route("get", "/settings/access-tokens", "系统设置", "查询管理 Access Token", "管理员查询已创建的服务访问令牌摘要。"), route("post", "/settings/access-tokens", "系统设置", "创建管理 Access Token", "创建随机服务 Bearer Token；完整 Token 仅在创建响应中返回。", { successStatus: 201 }), route("delete", "/settings/access-tokens/{token}", "系统设置", "删除管理 Access Token", "删除指定 Token，删除后立即失效。"), @@ -739,10 +739,10 @@ export const OPENAPI_DOCUMENT = { inbound_topic: textField("Agent 入站 Topic。"), outbound_topic: textField("Agent 出站 Topic。"), users: { type: "array", items: { type: "string" }, description: "当前目录项关联的用户列表。" }, - is_public: booleanField("是否为“所有人”可用的 Agent。") + is_public: booleanField("是否为“全部用户”可用的 Agent。") }, "外部 Agent 目录项。"), ExternalAgentSyncResponse: objectSchema({ - sessionId: textField("请求会话 ID。"), + sessionId: textField("服务端生成的一次性 MQTT 关联 ID；仅用于本次响应关联，不持久化到 external_sessions。", { readOnly: true }), kind: textField("固定为 final；不会返回 block 中间结果。", { enum: ["final"] }), text: textField("Agent 最终回复文本。") }, "同步 Agent 调用的最终结果。"), diff --git a/packages/daedalus-agent-management-web/server/src/services/agent-service.js b/packages/daedalus-agent-management-web/server/src/services/agent-service.js index 2a55a48..e8d9fa6 100644 --- a/packages/daedalus-agent-management-web/server/src/services/agent-service.js +++ b/packages/daedalus-agent-management-web/server/src/services/agent-service.js @@ -956,11 +956,14 @@ export class AgentService { rows = this.db.prepare(` SELECT ag.*, CASE WHEN COALESCE(policy.mode, 'selected') = 'all' THEN 1 ELSE 0 END AS is_public FROM agents ag - JOIN agent_users d LEFT JOIN agent_user_policies policy ON policy.agent_id = ag.id - LEFT JOIN agent_permissions ap ON ap.agent_id = ag.id AND ap.agent_user_id = d.id - WHERE d.username = ? - AND (COALESCE(policy.mode, 'selected') = 'all' OR ap.agent_id IS NOT NULL) + WHERE COALESCE(policy.mode, 'selected') = 'all' + OR EXISTS ( + SELECT 1 + FROM agent_users d + JOIN agent_permissions ap ON ap.agent_user_id = d.id + WHERE ap.agent_id = ag.id AND d.username = ? + ) ORDER BY ag.slug `).all(normalizedUsername); } catch (error) { diff --git a/packages/daedalus-agent-management-web/server/src/services/external-service.js b/packages/daedalus-agent-management-web/server/src/services/external-service.js index 9315374..d0753b1 100644 --- a/packages/daedalus-agent-management-web/server/src/services/external-service.js +++ b/packages/daedalus-agent-management-web/server/src/services/external-service.js @@ -104,12 +104,19 @@ export class ExternalService { if (username) { return Boolean(this.db.prepare(` SELECT 1 - FROM agent_users d - LEFT JOIN agent_permissions ap ON ap.agent_id = ? AND ap.agent_user_id = d.id - LEFT JOIN agent_user_policies policy ON policy.agent_id = ? - WHERE d.username = ? - AND (COALESCE(policy.mode, 'selected') = 'all' OR ap.agent_id IS NOT NULL) - `).get(agentId, agentId, username)); + FROM agents ag + LEFT JOIN agent_user_policies policy ON policy.agent_id = ag.id + WHERE ag.id = ? + AND ( + COALESCE(policy.mode, 'selected') = 'all' + OR EXISTS ( + SELECT 1 + FROM agent_users d + JOIN agent_permissions ap ON ap.agent_user_id = d.id + WHERE ap.agent_id = ag.id AND d.username = ? + ) + ) + `).get(agentId, username)); } return Boolean(this.db.prepare(` SELECT 1 @@ -165,6 +172,33 @@ export class ExternalService { return this.agentService.listAgentDirectoryForUser(normalizedUsername); } + ensureAgentUser(username) { + const now = toChinaISOString(); + try { + this.db.prepare(` + INSERT INTO agent_users (username, created_at, updated_at) + VALUES (?, ?, ?) + ON CONFLICT(username) DO NOTHING + `).run(username, now, now); + } catch (error) { + // 兼容仅保留 username 的旧测试/迁移数据库。 + if (!String(error?.message || "").toLowerCase().includes("no such column")) { + throw error; + } + this.db.prepare(` + INSERT INTO agent_users (username) + VALUES (?) + ON CONFLICT(username) DO NOTHING + `).run(username); + } + + return this.db.prepare(` + SELECT id, username + FROM agent_users + WHERE username = ? + `).get(username); + } + getOrCreateSession({ userName, agentId, cookie }) { const normalizedUserName = String(userName || "").trim(); const normalizedAgentId = String(agentId || "").trim(); @@ -193,13 +227,16 @@ export class ExternalService { throw notFound(`用户 ${normalizedUserName} 无权访问数字员工 ${normalizedAgentId}`); } - const agentUser = this.db.prepare(` + let agentUser = this.db.prepare(` SELECT id FROM agent_users WHERE username = ? `).get(normalizedUserName); if (!agentUser) { - throw notFound(`用户 ${normalizedUserName} 无权访问数字员工 ${normalizedAgentId}`); + agentUser = this.ensureAgentUser(normalizedUserName); + } + if (!agentUser) { + throw internalError(`无法为用户 ${normalizedUserName} 创建外部会话`); } const { inboundTopic, outboundTopic } = extractAgentTopics(agent.remote_state_json); diff --git a/packages/daedalus-agent-management-web/server/src/services/runtime-env-config-service.js b/packages/daedalus-agent-management-web/server/src/services/runtime-env-config-service.js index 29b195e..13f9130 100644 --- a/packages/daedalus-agent-management-web/server/src/services/runtime-env-config-service.js +++ b/packages/daedalus-agent-management-web/server/src/services/runtime-env-config-service.js @@ -1,3 +1,4 @@ +import crypto from "node:crypto"; import fs from "node:fs"; import { isIP } from "node:net"; import path from "node:path"; @@ -19,6 +20,10 @@ function firstText(...values) { return ""; } +function runtimeCredentialSecret() { + return crypto.randomBytes(24).toString("base64url"); +} + function normalizeAllowlistValue(value) { if (value === undefined || value === null) { return ""; @@ -246,6 +251,38 @@ export class RuntimeEnvConfigService { }; } + resetRuntimeCredentialPasswords() { + const payload = readJsonObject(this.credentialsPath, "credentials.json"); + const mqtt = payload.mqtt && typeof payload.mqtt === "object" && !Array.isArray(payload.mqtt) + ? payload.mqtt + : {}; + const minio = payload.minio && typeof payload.minio === "object" && !Array.isArray(payload.minio) + ? payload.minio + : {}; + const nextPayload = { + ...payload, + mqtt: { + ...mqtt, + username: firstText(mqtt.username, "daedalus-agent"), + password: runtimeCredentialSecret() + }, + minio: { + ...minio, + username: firstText(minio.username, "daedalus-agent"), + password: runtimeCredentialSecret() + }, + generatedAt: payload.generatedAt || toChinaISOString() + }; + + writeJsonObject(this.credentialsPath, nextPayload); + return { + mqtt_broker_username: nextPayload.mqtt.username, + mqtt_broker_password: nextPayload.mqtt.password, + s3_api_username: nextPayload.minio.username, + s3_api_password: nextPayload.minio.password + }; + } + updateNetworkAllowlistConfig(payload) { if (!this.db) { throw new Error("Runtime settings database is not configured"); diff --git a/packages/daedalus-agent-management-web/server/src/services/topic-ping-service.js b/packages/daedalus-agent-management-web/server/src/services/topic-ping-service.js index 9664f3e..1b68ba3 100644 --- a/packages/daedalus-agent-management-web/server/src/services/topic-ping-service.js +++ b/packages/daedalus-agent-management-web/server/src/services/topic-ping-service.js @@ -9,6 +9,10 @@ const DEFAULT_TIMEOUT_MS = 15000; const SYNC_MESSAGE_TIMEOUT_MS = 120000; const BLOCK_STREAM_IDLE_MS = 1500; +function createTransientSyncSessionId() { + return `external-sync-${randomUUID()}`; +} + function firstText(...values) { for (const value of values) { if (typeof value === "string" && value.trim()) { @@ -355,17 +359,12 @@ export class TopicPingService { } async syncAgentMessage(agentId, { - sessionId, senderId, text, timeoutMs = SYNC_MESSAGE_TIMEOUT_MS } = {}) { - const normalizedSessionId = String(sessionId || "").trim(); const normalizedSenderId = String(senderId || "").trim(); const normalizedText = String(text || "").trim(); - if (!normalizedSessionId) { - throw badRequest("sessionId 不能为空"); - } if (!normalizedSenderId) { throw badRequest("senderId 不能为空"); } @@ -378,9 +377,10 @@ export class TopicPingService { const normalizedAgentId = String(agentId || "").trim(); const { inboundTopic, outboundTopic } = this.resolveAgentTopics(normalizedAgentId); + const transientSessionId = createTransientSyncSessionId(); const safeTimeoutMs = normalizeTimeoutMs(timeoutMs, SYNC_MESSAGE_TIMEOUT_MS); const payload = { - sessionId: normalizedSessionId, + sessionId: transientSessionId, senderId: normalizedSenderId, disableBlockStreaming: true, text: normalizedText @@ -459,7 +459,7 @@ export class TopicPingService { cleanup(); reject(serviceUnavailable(`等待数字员工 ${normalizedAgentId} 最终回复超时`, { timeoutMs: safeTimeoutMs, - sessionId: normalizedSessionId, + sessionId: transientSessionId, inboundTopic, outboundTopic })); @@ -483,7 +483,7 @@ export class TopicPingService { if ( !message || typeof message !== "object" - || String(message.sessionId || "").trim() !== normalizedSessionId + || String(message.sessionId || "").trim() !== transientSessionId || message.kind !== "final" ) { return; @@ -495,7 +495,7 @@ export class TopicPingService { cancelReplyWait = () => { cleanup(); reject(serviceUnavailable("已停止等待数字员工 MQTT 最终回复", { - sessionId: normalizedSessionId, + sessionId: transientSessionId, inboundTopic, outboundTopic })); diff --git a/packages/daedalus-agent-management-web/server/test/agent-user-policy.test.js b/packages/daedalus-agent-management-web/server/test/agent-user-policy.test.js index 534d113..1887c75 100644 --- a/packages/daedalus-agent-management-web/server/test/agent-user-policy.test.js +++ b/packages/daedalus-agent-management-web/server/test/agent-user-policy.test.js @@ -63,12 +63,19 @@ it("applies all-user access to current and future agent users", async () => { expect(policy.mode).toBe("all"); expect(policy.policy_version).toBe(2); expect(await service.listAgentDirectoryForUser("bob")).toHaveSize(1); + expect(await service.listAgentDirectoryForUser("outside-directory-user")).toHaveSize(1); service.createAgentUser("carol"); expect(await service.listAgentDirectoryForUser("carol")).toHaveSize(1); expect(db.prepare("SELECT COUNT(*) AS count FROM agent_permissions WHERE agent_id = 1").get().count).toBe(0); }); +it("does not expose selected agents to usernames outside the agent directory", async () => { + const service = createService(createDb()); + + expect(await service.listAgentDirectoryForUser("outside-directory-user")).toEqual([]); +}); + it("lists only all-user agents as public directory entries", async () => { const db = createDb(); db.prepare("INSERT INTO agents (id, slug, agent_name, status) VALUES (?, ?, ?, ?)") diff --git a/packages/daedalus-agent-management-web/server/test/app-smoke.test.js b/packages/daedalus-agent-management-web/server/test/app-smoke.test.js index 5b11045..383b4ea 100644 --- a/packages/daedalus-agent-management-web/server/test/app-smoke.test.js +++ b/packages/daedalus-agent-management-web/server/test/app-smoke.test.js @@ -54,7 +54,7 @@ it("assembles the management server and maps application and parser errors", asy jasmine.objectContaining({ type: "boolean" }) ); expect(docsJson.paths["/external/agents/{agentId}/call"].post.requestBody.content["application/json"].schema.required) - .toEqual(["sessionId", "senderId", "text"]); + .toEqual(["senderId", "text"]); expect(docsJson.components.schemas.ExternalAgentSyncResponse.properties.kind.enum).toEqual(["final"]); const badLogin = await fetch(`${baseUrl}/api/auth/login`, { @@ -121,7 +121,7 @@ it("handles synchronous agent requests and returns only the final result", async services.topicPingService.syncAgentMessage = async (agentId, payload) => { calls.push({ agentId, payload }); return { - sessionId: payload.sessionId, + sessionId: `external-sync-${calls.length}`, kind: "final", text: "同步结果" }; @@ -141,24 +141,24 @@ it("handles synchronous agent requests and returns only the final result", async const response = await fetch(`${baseUrl}/api/external/agents/finance-agent/call`, { method: "POST", headers: { "content-type": "application/json" }, - body: JSON.stringify({ sessionId: "session-1", senderId: "sender-1", text: "hello" }) + body: JSON.stringify({ sessionId: "caller-provided-session", senderId: "sender-1", text: "hello" }) }); expect(response.status).toBe(200); expect(await response.json()).toEqual({ - sessionId: "session-1", + sessionId: "external-sync-1", kind: "final", text: "同步结果" }); expect(calls).toEqual([{ agentId: "finance-agent", - payload: { sessionId: "session-1", senderId: "sender-1", text: "hello" } + payload: { senderId: "sender-1", text: "hello" } }]); expect(usageRefreshes).toEqual([{ trigger: "external-agent-call" }]); const invalidResponse = await fetch(`${baseUrl}/api/external/agents/finance-agent/call`, { method: "POST", headers: { "content-type": "application/json" }, - body: JSON.stringify({ sessionId: "session-1", senderId: "sender-1" }) + body: JSON.stringify({ senderId: "sender-1" }) }); expect(invalidResponse.status).toBe(400); expect(usageRefreshes).toHaveSize(1); @@ -167,11 +167,11 @@ it("handles synchronous agent requests and returns only the final result", async const failedRefreshResponse = await fetch(`${baseUrl}/api/external/agents/finance-agent/call`, { method: "POST", headers: { "content-type": "application/json" }, - body: JSON.stringify({ sessionId: "session-2", senderId: "sender-1", text: "hello again" }) + body: JSON.stringify({ senderId: "sender-1", text: "hello again" }) }); expect(failedRefreshResponse.status).toBe(200); expect(await failedRefreshResponse.json()).toEqual({ - sessionId: "session-2", + sessionId: "external-sync-2", kind: "final", text: "同步结果" }); @@ -180,11 +180,11 @@ it("handles synchronous agent requests and returns only the final result", async const thrownRefreshResponse = await fetch(`${baseUrl}/api/external/agents/finance-agent/call`, { method: "POST", headers: { "content-type": "application/json" }, - body: JSON.stringify({ sessionId: "session-3", senderId: "sender-1", text: "hello once more" }) + body: JSON.stringify({ senderId: "sender-1", text: "hello once more" }) }); expect(thrownRefreshResponse.status).toBe(200); expect(await thrownRefreshResponse.json()).toEqual({ - sessionId: "session-3", + sessionId: "external-sync-3", kind: "final", text: "同步结果" }); diff --git a/packages/daedalus-agent-management-web/server/test/external-service.test.js b/packages/daedalus-agent-management-web/server/test/external-service.test.js index 5bbae9f..723cc08 100644 --- a/packages/daedalus-agent-management-web/server/test/external-service.test.js +++ b/packages/daedalus-agent-management-web/server/test/external-service.test.js @@ -17,7 +17,9 @@ function createDb() { CREATE TABLE agent_users ( id INTEGER PRIMARY KEY, - username TEXT NOT NULL UNIQUE + username TEXT NOT NULL UNIQUE, + created_at TEXT NOT NULL DEFAULT '', + updated_at TEXT NOT NULL DEFAULT '' ); CREATE TABLE agent_permissions ( @@ -191,6 +193,57 @@ it("allows current and future agent users when the user policy is all", () => { expect(() => service.resolveAuthorizedSystem(session.sessionId, "crm-app")).not.toThrow(); }); +it("allows external sessions for usernames outside the agent directory in all-user mode", () => { + const db = createDb(); + db.exec(` + CREATE TABLE agent_user_policies ( + agent_id INTEGER PRIMARY KEY, + mode TEXT NOT NULL, + policy_version INTEGER NOT NULL, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL + ); + INSERT INTO agent_user_policies ( + agent_id, mode, policy_version, created_at, updated_at + ) VALUES (1, 'all', 1, '2026-05-26T00:00:00.000Z', '2026-05-26T00:00:00.000Z'); + `); + + const service = createService(db); + const session = service.getOrCreateSession({ + userName: "outside-directory-user", + agentId: "agent-a" + }); + + expect(session.sessionId).toMatch(/^s-[1-9]{30}$/); + expect(db.prepare("SELECT id FROM agent_users WHERE username = ?").get("outside-directory-user")) + .toEqual(jasmine.objectContaining({ id: jasmine.any(Number) })); + expect(() => service.resolveAuthorizedSystem(session.sessionId, "crm-app")).not.toThrow(); +}); + +it("keeps outside-directory usernames unauthorized in selected-user mode", () => { + const db = createDb(); + db.exec(` + CREATE TABLE agent_user_policies ( + agent_id INTEGER PRIMARY KEY, + mode TEXT NOT NULL, + policy_version INTEGER NOT NULL, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL + ); + INSERT INTO agent_user_policies ( + agent_id, mode, policy_version, created_at, updated_at + ) VALUES (1, 'selected', 1, '2026-05-26T00:00:00.000Z', '2026-05-26T00:00:00.000Z'); + `); + + const service = createService(db); + expect(() => service.getOrCreateSession({ + userName: "outside-directory-user", + agentId: "agent-a" + })).toThrowError(/无权访问/); + expect(db.prepare("SELECT id FROM agent_users WHERE username = ?").get("outside-directory-user")) + .toBeUndefined(); +}); + it("returns topics from catalog snapshot shaped remote state", () => { const db = createDb(); db.prepare(` diff --git a/packages/daedalus-agent-management-web/server/test/runtime-env-config-service.test.js b/packages/daedalus-agent-management-web/server/test/runtime-env-config-service.test.js index fe5f4e4..c5d3d50 100644 --- a/packages/daedalus-agent-management-web/server/test/runtime-env-config-service.test.js +++ b/packages/daedalus-agent-management-web/server/test/runtime-env-config-service.test.js @@ -274,6 +274,55 @@ it("reads mqtt and s3 credentials from credentials.json", () => { } }); +it("resets mqtt and s3 passwords while preserving credential metadata", () => { + const tempDir = createTempDir(); + const credentialsPath = path.join(tempDir, "credentials.json"); + fs.writeFileSync(credentialsPath, JSON.stringify({ + mqtt: { + username: "mqtt-user", + password: "old-mqtt-pass", + clientId: "management" + }, + minio: { + username: "s3-user", + password: "old-s3-pass", + region: "local" + }, + generatedAt: "2026-07-01T00:00:00+08:00", + version: 1 + }, null, 2)); + + try { + const service = new RuntimeEnvConfigService({ credentialsPath }); + const result = service.resetRuntimeCredentialPasswords(); + const saved = JSON.parse(fs.readFileSync(credentialsPath, "utf8")); + + expect(result.mqtt_broker_username).toBe("mqtt-user"); + expect(result.s3_api_username).toBe("s3-user"); + expect(result.mqtt_broker_password).toMatch(/^[A-Za-z0-9_-]{32}$/); + expect(result.s3_api_password).toMatch(/^[A-Za-z0-9_-]{32}$/); + expect(result.mqtt_broker_password).not.toBe("old-mqtt-pass"); + expect(result.s3_api_password).not.toBe("old-s3-pass"); + expect(result.mqtt_broker_password).not.toBe(result.s3_api_password); + expect(saved).toEqual({ + mqtt: { + username: "mqtt-user", + password: result.mqtt_broker_password, + clientId: "management" + }, + minio: { + username: "s3-user", + password: result.s3_api_password, + region: "local" + }, + generatedAt: "2026-07-01T00:00:00+08:00", + version: 1 + }); + } finally { + fs.rmSync(tempDir, { recursive: true, force: true }); + } +}); + it("writes a container restart request file", () => { const tempDir = createTempDir(); const restartRequestPath = path.join(tempDir, "run", "container-restart.request"); diff --git a/packages/daedalus-agent-management-web/server/test/runtime-settings-route.test.js b/packages/daedalus-agent-management-web/server/test/runtime-settings-route.test.js index b699b9e..aac8b4b 100644 --- a/packages/daedalus-agent-management-web/server/test/runtime-settings-route.test.js +++ b/packages/daedalus-agent-management-web/server/test/runtime-settings-route.test.js @@ -281,6 +281,68 @@ it("returns runtime component health", async () => { } }); +it("resets runtime credential passwords, requests a container restart, and writes an audit log", async () => { + const auditWrites = []; + const resetCalls = []; + const restartRequests = []; + const credentials = { + mqtt_broker_username: "mqtt-user", + mqtt_broker_password: "new-mqtt-pass", + s3_api_username: "s3-user", + s3_api_password: "new-s3-pass" + }; + const app = express(); + app.use(express.json()); + app.use("/api", createRoutes(createBaseServices({ + auditLogService: { + write(entry) { + auditWrites.push(entry); + } + }, + runtimeEnvConfigService: { + resetRuntimeCredentialPasswords() { + resetCalls.push({}); + return credentials; + } + }, + containerService: { + requestRestart(options) { + restartRequests.push(options); + return { + requested: true, + requested_at: "2026-07-01T00:00:00.000Z" + }; + } + } + }))); + + const server = await listen(app); + try { + const address = server.address(); + const response = await requestJson( + `http://127.0.0.1:${address.port}/api/settings/runtime-credentials/reset-password`, + { + method: "POST", + body: {} + } + ); + + expect(response.status).toBe(200); + expect(response.body.credentials).toEqual(credentials); + expect(response.body.restart.requested).toBeTrue(); + expect(resetCalls).toEqual([{}]); + expect(restartRequests).toEqual([{ requestedBy: "daedalus-agent" }]); + expect(auditWrites).toEqual([{ + userId: 1, + username: "daedalus-agent", + action: "重置运行凭据密码", + detail: "重置 MQTT/MinIO 运行凭据密码并请求重启容器" + }]); + } finally { + await close(server); + } +}); + it("requests a container restart without calling the gateway restart action", async () => { const auditWrites = []; const restartRequests = []; diff --git a/packages/daedalus-agent-management-web/server/test/topic-ping-service.test.js b/packages/daedalus-agent-management-web/server/test/topic-ping-service.test.js index c693917..fde7c9d 100644 --- a/packages/daedalus-agent-management-web/server/test/topic-ping-service.test.js +++ b/packages/daedalus-agent-management-web/server/test/topic-ping-service.test.js @@ -278,11 +278,13 @@ it("sends a single agent test message and returns MQTT payloads with timings", a expect(result.timings.total_ms).toEqual(jasmine.any(Number)); }); -it("sends a synchronous agent message with block streaming disabled and returns only final", async () => { +it("creates a transient session internally, sends no session input, and returns only final", async () => { const client = new MockMqttClient(); + const statements = []; const service = new TopicPingService({ db: { - prepare() { + prepare(statement) { + statements.push(statement); return { get() { return { @@ -330,42 +332,37 @@ it("sends a synchronous agent message with block streaming disabled and returns }; const result = await service.syncAgentMessage("agent-a", { - sessionId: "session-1", senderId: "sender-1", text: "hello" }); expect(result).toEqual({ - sessionId: "session-1", + sessionId: jasmine.stringMatching(/^external-sync-/), kind: "final", text: "final answer" }); + expect(client.publications[0].payload.sessionId).toBe(result.sessionId); expect(client.publications[0]).toEqual({ topic: "agent/agent-a/in", payload: { - sessionId: "session-1", + sessionId: jasmine.stringMatching(/^external-sync-/), senderId: "sender-1", disableBlockStreaming: true, text: "hello" } }); + expect(statements.every((statement) => !/INSERT|UPDATE|DELETE|external_sessions|external_session_cookies/i.test(statement))) + .toBeTrue(); }); it("validates synchronous agent message fields", async () => { const service = createTopicService(); await expectAsync(service.syncAgentMessage("agent-a", { - sessionId: "", - senderId: "sender-1", - text: "hello" - })).toBeRejectedWithError(/sessionId/); - await expectAsync(service.syncAgentMessage("agent-a", { - sessionId: "session-1", senderId: "", text: "hello" })).toBeRejectedWithError(/senderId/); await expectAsync(service.syncAgentMessage("agent-a", { - sessionId: "session-1", senderId: "sender-1", text: " " })).toBeRejectedWithError(/文本/); diff --git a/packages/daedalus-agent-management-web/src/app/App.jsx b/packages/daedalus-agent-management-web/src/app/App.jsx index 8958728..86f94e3 100644 --- a/packages/daedalus-agent-management-web/src/app/App.jsx +++ b/packages/daedalus-agent-management-web/src/app/App.jsx @@ -474,6 +474,17 @@ export default function App() { onContainerRestart={async () => { return await api.post("/api/settings/container-restart", {}); }} + onResetRuntimeCredentialPasswords={async () => { + const next = await api.post("/api/settings/runtime-credentials/reset-password", {}); + setBoot((current) => ({ + ...current, + env_config: { + ...(current?.env_config || {}), + ...(next?.credentials || {}) + } + })); + return next; + }} onRefreshRuntimeHealth={refreshRuntimeHealth} onExecuteCommand={async ({ action, params }) => { return await api.post("/api/settings/command", { action, params }); diff --git a/packages/daedalus-agent-management-web/src/pages/AgentsPage.jsx b/packages/daedalus-agent-management-web/src/pages/AgentsPage.jsx index ab00b8d..a8d9067 100644 --- a/packages/daedalus-agent-management-web/src/pages/AgentsPage.jsx +++ b/packages/daedalus-agent-management-web/src/pages/AgentsPage.jsx @@ -1416,11 +1416,11 @@ export function AgentsPage({ refreshKey = 0, createTemplateName = "", onCreateTe {permissionMode === "all" ? ( - + ) : null} {permissionMode !== "all" ? ( {detailAgent.user_policy?.mode === "all" - ? 所有人 + ? 全部用户 : (detailAgent.permissions || []).length > 0 ? detailAgent.permissions.map((item) => ( diff --git a/packages/daedalus-agent-management-web/src/pages/SettingsPage.jsx b/packages/daedalus-agent-management-web/src/pages/SettingsPage.jsx index 9700d37..7cef874 100644 --- a/packages/daedalus-agent-management-web/src/pages/SettingsPage.jsx +++ b/packages/daedalus-agent-management-web/src/pages/SettingsPage.jsx @@ -539,6 +539,7 @@ export function SettingsPage({ onCatalogSync, onServerRestart, onContainerRestart, + onResetRuntimeCredentialPasswords, onRefreshRuntimeHealth, onExecuteCommand }) { @@ -561,6 +562,8 @@ export function SettingsPage({ const [containerRestartConfirmOpen, setContainerRestartConfirmOpen] = useState(false); const [containerRestartConfirmTitle, setContainerRestartConfirmTitle] = useState("确认重启容器？"); const [containerRestartConfirmContent, setContainerRestartConfirmContent] = useState("重启容器会导致管理控制台短暂不可访问，确认后将立即重启整个容器。"); + const [runtimeCredentialResetting, setRuntimeCredentialResetting] = useState(false); + const [runtimeCredentialResetConfirmOpen, setRuntimeCredentialResetConfirmOpen] = useState(false); const [logoUploading, setLogoUploading] = useState(false); const [logoClearing, setLogoClearing] = useState(false); const [logoPreviewSrc, setLogoPreviewSrc] = useState(logoSrc || ""); @@ -657,6 +660,21 @@ export function SettingsPage({ } }; + const resetRuntimeCredentialPasswords = async () => { + setRuntimeCredentialResetting(true); + try { + await runWithOperationMask("正在重置运行凭据密码，请稍候...", async () => { + await onResetRuntimeCredentialPasswords(); + }); + setRuntimeCredentialResetConfirmOpen(false); + notify?.success("运行凭据密码已重置，容器重启请求已发送"); + } catch (error) { + notify?.error(`运行凭据密码重置失败：${getErrorMessage(error, "重置失败")}`); + } finally { + setRuntimeCredentialResetting(false); + } + }; + const runWithOperationMask = async (text, action) => { setOperationMaskText(text); setOperationMaskOpen(true); @@ -942,6 +960,27 @@ export function SettingsPage({ > {containerRestartConfirmContent} + , "data-testid": "settings-runtime-credentials-reset-confirm-submit" }} + cancelButtonProps={{ icon: , "data-testid": "settings-runtime-credentials-reset-confirm-cancel" }} + onCancel={() => { + if (runtimeCredentialResetting) { + return; + } + setRuntimeCredentialResetConfirmOpen(false); + }} + onOk={() => { + void resetRuntimeCredentialPasswords(); + }} + > + MQTT 和 S3 运行凭据密码都会被替换，并立即请求重启容器；重启期间管理控制台会短暂不可访问。 + } - loading={runtimeHealthLoading} - onClick={() => { - void onRefreshRuntimeHealth?.({ force: true }); - }} - > - 刷新 - + + + + )} > {assignmentMode === "all" ? ( - + ) : ( <> + + + + diff --git a/packages/daedalus-agent-management-web/src/pages/AgentUsersPage.test.jsx b/packages/daedalus-agent-management-web/src/pages/AgentUsersPage.test.jsx new file mode 100644 index 0000000..79df1bf --- /dev/null +++ b/packages/daedalus-agent-management-web/src/pages/AgentUsersPage.test.jsx @@ -0,0 +1,42 @@ +// @vitest-environment jsdom +import React from "react"; +import { cleanup, fireEvent, render, screen, waitFor } from "@testing-library/react"; +import { afterEach, describe, expect, it, vi } from "vitest"; + +import { AgentUsersPage } from "./AgentUsersPage.jsx"; +import { api } from "../app/api-client.js"; + +vi.mock("../app/api-client.js", () => ({ + api: { get: vi.fn(), post: vi.fn(), put: vi.fn(), delete: vi.fn() } +})); + +afterEach(() => { + cleanup(); + vi.clearAllMocks(); +}); + +it("shows quota columns and edit action, and submits the default unlimited quota for new users", async () => { + api.get.mockResolvedValue({ + items: [{ id: 1, username: "alice", daily_limit: -1, today_tokens: 0, status: "normal", assigned_agents: 0, assigned_agent_items: [] }], + total: 1, + page: 1, + pageSize: 10 + }); + api.post.mockResolvedValue({ total: 1, created: 1 }); + + render(); + expect(await screen.findByText("每日限额")).toBeInTheDocument(); + expect(screen.getByText("今日用量")).toBeInTheDocument(); + expect(screen.getByTestId("agent-user-edit-alice")).toBeInTheDocument(); + + fireEvent.click(screen.getByTestId("agent-user-create-button")); + const quotaInput = await screen.findByTestId("agent-user-form-daily-limit"); + expect(quotaInput).toHaveValue("-1"); + fireEvent.change(screen.getByTestId("agent-user-form-content"), { target: { value: "bob" } }); + fireEvent.click(screen.getByTestId("agent-user-form-submit")); + + await waitFor(() => expect(api.post).toHaveBeenCalledWith("/api/agent-users", { + username: "bob", + daily_limit: -1 + })); +}); diff --git a/packages/daedalus-agent-management-web/src/pages/DashboardPage.jsx b/packages/daedalus-agent-management-web/src/pages/DashboardPage.jsx index 25d522b..cd3cbab 100644 --- a/packages/daedalus-agent-management-web/src/pages/DashboardPage.jsx +++ b/packages/daedalus-agent-management-web/src/pages/DashboardPage.jsx @@ -72,9 +72,18 @@ export function DashboardPage({ refreshKey = 0 }) { return ; } - const sortedAgentUsage = [...(data.agentUsage || [])].sort((current, next) => ( - Number(next.today_tokens || 0) - Number(current.today_tokens || 0) - )); + const sortedAgentUsage = [...(data.agentUsage || [])] + .filter((item) => Number(item.today_tokens || 0) > 0) + .sort((current, next) => ( + Number(next.today_tokens || 0) - Number(current.today_tokens || 0) + || String(current.slug || "").localeCompare(String(next.slug || "")) + )); + const sortedUserUsage = [...(data.userUsage || [])] + .filter((item) => Number(item.today_tokens || 0) > 0) + .sort((current, next) => ( + Number(next.today_tokens || 0) - Number(current.today_tokens || 0) + || String(current.username || "").localeCompare(String(next.username || "")) + )); return ( @@ -105,12 +114,12 @@ export function DashboardPage({ refreshKey = 0 }) { ))} - +
+ +
({ + "data-testid": `dashboard-user-usage-row-${row.username}` + })} + columns={[ + { title: "用户名", dataIndex: "username" }, + { + title: "状态", + dataIndex: "status", + render: (value) => {value === "overlimit" ? "已超限" : "正常"} + }, + { title: "每日限额", dataIndex: "daily_limit", render: formatQuota }, + { title: "今日消耗", dataIndex: "today_tokens", render: formatTokenCount } + ]} + /> + ); } diff --git a/packages/daedalus-agent-management-web/src/pages/DashboardPage.test.jsx b/packages/daedalus-agent-management-web/src/pages/DashboardPage.test.jsx new file mode 100644 index 0000000..3f4b647 --- /dev/null +++ b/packages/daedalus-agent-management-web/src/pages/DashboardPage.test.jsx @@ -0,0 +1,40 @@ +// @vitest-environment jsdom +import React from "react"; +import { cleanup, render, screen } from "@testing-library/react"; +import { afterEach, describe, expect, it, vi } from "vitest"; + +import { DashboardPage } from "./DashboardPage.jsx"; +import { api } from "../app/api-client.js"; + +vi.mock("../app/api-client.js", () => ({ api: { get: vi.fn() } })); + +afterEach(() => { + cleanup(); + vi.clearAllMocks(); +}); + +it("renders separate ten-row agent and user usage tables ordered by today's usage", async () => { + api.get.mockResolvedValue({ + stats: { agents: 1, templates: 1, skills: 1, systems: 1, today_tokens: 300 }, + recentInvocations: [], + agentUsage: Array.from({ length: 11 }, (_, index) => ({ + id: index + 1, + slug: `agent-${index + 1}`, + agent_name: `Agent ${index + 1}`, + status: "normal", + daily_limit: -1, + today_tokens: 110 - index * 10 + })), + userUsage: [ + { id: 1, username: "alice", status: "normal", daily_limit: -1, today_tokens: 80 }, + { id: 2, username: "bob", status: "normal", daily_limit: -1, today_tokens: 10 } + ] + }); + + render(); + expect(await screen.findByText("用量表（用户）")).toBeInTheDocument(); + expect(screen.getByText("用量表（用户维度）")).toBeInTheDocument(); + expect(screen.getByTestId("dashboard-agent-usage-row-agent-1")).toBeInTheDocument(); + expect(screen.queryByTestId("dashboard-agent-usage-row-agent-11")).not.toBeInTheDocument(); + expect(screen.getByTestId("dashboard-user-usage-row-alice")).toBeInTheDocument(); +}); diff --git a/packages/daedalus-agent-management-web/test/backend-api-regression.ts b/packages/daedalus-agent-management-web/test/backend-api-regression.ts index f389a4e..f1ed704 100644 --- a/packages/daedalus-agent-management-web/test/backend-api-regression.ts +++ b/packages/daedalus-agent-management-web/test/backend-api-regression.ts @@ -289,6 +289,26 @@ async function main(): Promise { expectStatus(await client.request("DELETE", "/api/agents/99999", { json: {} }), 404, "delete missing agent"); expectStatus(await client.request("GET", "/api/agent-users?page=1&limit=50"), 200, "list agent users"); expectStatus(await client.request("POST", "/api/agent-users", { json: {} }), 400, "invalid agent user"); + const quotaAgentUser = await client.request<{ id: number; username: string; daily_limit: number }>("POST", "/api/agent-users", { + json: { username: "quota-regression-user", daily_limit: 1234 } + }); + expectStatus(quotaAgentUser, 201, "create quota agent user"); + assert.equal(quotaAgentUser.body.daily_limit, 1234); + const updatedQuotaAgentUser = await client.request<{ username: string; daily_limit: number }>("PUT", `/api/agent-users/${quotaAgentUser.body.id}`, { + json: { username: "quota-regression-user-renamed", daily_limit: 2345 } + }); + expectStatus(updatedQuotaAgentUser, 200, "update quota agent user"); + assert.equal(updatedQuotaAgentUser.body.username, "quota-regression-user-renamed"); + assert.equal(updatedQuotaAgentUser.body.daily_limit, 2345); + expectStatus(await client.request("DELETE", `/api/agent-users/${quotaAgentUser.body.id}`), 200, "delete quota agent user"); + const importedQuotaUsers = await client.request<{ created: number }>("POST", "/api/agent-users/import", { + json: { content: "quota-import-user-1\nquota-import-user-2", daily_limit: 3456 } + }); + expectStatus(importedQuotaUsers, 200, "import quota agent users"); + assert.equal(importedQuotaUsers.body.created, 2); + const listedQuotaUsers = await client.request<{ items: Array<{ username: string; daily_limit: number }> }>("GET", "/api/agent-users?page=1&limit=200&q=quota-import-user"); + expectStatus(listedQuotaUsers, 200, "list imported quota agent users"); + assert.deepEqual(listedQuotaUsers.body.items.map((item) => item.daily_limit), [3456, 3456]); expectStatus(await client.request("POST", "/api/agent-users/import", { json: {} }), 400, "invalid agent user import"); expectStatus(await client.request("DELETE", "/api/agent-users/99999"), 404, "delete missing agent user"); expectStatus(await client.request("GET", "/api/mcp-servers"), 200, "list mcp servers"); diff --git a/packages/daedalus-agent-management-web/test/run-browser-e2e.js b/packages/daedalus-agent-management-web/test/run-browser-e2e.js index 4f65fd6..6a91cda 100644 --- a/packages/daedalus-agent-management-web/test/run-browser-e2e.js +++ b/packages/daedalus-agent-management-web/test/run-browser-e2e.js @@ -30,9 +30,9 @@ function findChromeExecutable() { throw new Error("No Chrome/Edge executable found"); } -function jsonBody(payload) { +function jsonBody(payload, status = 200) { return { - status: 200, + status, contentType: "application/json", body: JSON.stringify(payload) }; @@ -439,11 +439,17 @@ async function main() { let agentUsers = [{ id: 1, username: "alice", + daily_limit: -1, + today_tokens: 0, + status: "normal", assigned_agents: 1, assigned_agent_items: [{ id: 1, slug: "finance-agent", agent_name: "E2E Agent" }] }, { id: 2, username: "bob", + daily_limit: -1, + today_tokens: 0, + status: "normal", assigned_agents: 0, assigned_agent_items: [] }]; @@ -779,7 +785,8 @@ async function main() { status: agent.status, daily_limit: agent.daily_limit, today_tokens: agent.usage?.daily || 0 - })) + })), + userUsage: agentUsers.filter((user) => Number(user.today_tokens || 0) > 0) })); return; } @@ -807,6 +814,9 @@ async function main() { ...created.map((username, index) => ({ id: agentUsers.length + index + 1, username, + daily_limit: Number(payload.daily_limit ?? -1), + today_tokens: 0, + status: "normal", assigned_agents: 0, assigned_agent_items: [] })) @@ -815,6 +825,34 @@ async function main() { return; } + if (pathname === "/api/agent-users" && method === "POST") { + const payload = JSON.parse(request.postData() || "{}"); + const nextId = Math.max(0, ...agentUsers.map((item) => item.id)) + 1; + agentUsers.push({ + id: nextId, + username: payload.username, + daily_limit: Number(payload.daily_limit ?? -1), + today_tokens: 0, + status: "normal", + assigned_agents: 0, + assigned_agent_items: [] + }); + await route.fulfill(jsonBody(agentUsers.at(-1), 201)); + return; + } + + if (/^\/api\/agent-users\/\d+$/.test(pathname) && method === "PUT") { + const payload = JSON.parse(request.postData() || "{}"); + const id = Number(pathname.split("/").at(-1)); + const user = agentUsers.find((item) => item.id === id); + if (user) { + user.username = payload.username ?? user.username; + user.daily_limit = Number(payload.daily_limit ?? user.daily_limit ?? -1); + } + await route.fulfill(jsonBody(user || {}, 200)); + return; + } + if (/^\/api\/agent-users\/\d+$/.test(pathname) && method === "DELETE") { const id = Number(pathname.split("/").at(-1)); agentUsers = agentUsers.filter((item) => item.id !== id); @@ -1524,7 +1562,7 @@ async function main() { status: "normal", daily_limit: createPayload.daily_limit, permissions: [], - usage: { daily: 0 }, + usage: { daily: 120 }, health: "normal", skills: [] }; @@ -1857,6 +1895,7 @@ async function main() { await page.getByTestId("dashboard-metric-agents-value").waitFor({ state: "visible" }); assert.equal(await page.getByTestId("dashboard-metric-agents-value").innerText(), "1"); await page.getByTestId("dashboard-agent-usage-row-finance-agent").waitFor({ state: "visible" }); + assert.equal(await page.getByTestId("dashboard-user-usage-row-alice").count(), 0); const requestsAfterReturn = dashboardRequestCount; const dashboardRefresh = page.waitForResponse((response) => { @@ -2458,7 +2497,7 @@ async function main() { }); await page.getByTestId("settings-runtime-health-refresh-button").click(); await runtimeHealthResponse; - await page.getByRole("button", { name: "刷新状态", exact: true }).waitFor({ state: "visible" }); + await page.getByTestId("settings-runtime-health-refresh-button").waitFor({ state: "visible" }); await page.getByTestId("settings-runtime-credentials-reset-button").click(); await page.getByText("确认重置运行凭据密码？", { exact: true }).waitFor({ state: "visible" }); await page.getByText("MQTT 和 S3 运行凭据密码都会被替换，并立即请求重启容器；重启期间管理控制台会短暂不可访问。", { exact: true }).waitFor({ state: "visible" }); -- Gitee From 3694373489104fad92d01841f0371f8e42d8380f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=AE=81=E4=BC=9F?= Date: Wed, 2 Sep 2026 08:54:43 +0800 Subject: [PATCH 7/9] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E7=94=A8=E9=87=8F?= =?UTF-8?q?=E8=AE=A1=E6=95=B0=E7=9A=84bug?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- doc/system-feature-list.md | 6 +- docker/assets/config.json | 5 +- docker/assets/scripts/kernel-startup.sh | 1 + .../daedalus-agent-mqtt-channel/DEV.md | 4 + .../daedalus-agent-mqtt-channel/README.md | 16 +- .../daedalus-agent-mqtt-channel/index.test.ts | 18 ++ .../daedalus-agent-mqtt-channel/index.ts | 16 + .../src/model-usage.test.ts | 159 ++++++++++ .../src/model-usage.ts | 283 ++++++++++++++++++ .../test/run-jasmine.ts | 2 +- .../docs/spec.md | 2 +- .../server/src/api/routes/index.js | 11 + .../server/src/docs/openapi.js | 23 ++ .../server/src/services/external-service.js | 41 +++ .../server/src/services/portal-service.js | 2 +- .../server/test/app-smoke.test.js | 4 +- .../test/model-gateway-usage-route.test.js | 35 +++ .../test/portal-service-alignment.test.js | 47 +++ .../server/test/user-quota.test.js | 18 ++ .../src/pages/DashboardPage.jsx | 4 +- .../src/pages/DashboardPage.test.jsx | 4 +- .../upgrade/openclaw-upgrade-contract.test.js | 9 + 22 files changed, 697 insertions(+), 13 deletions(-) create mode 100644 extensions/openclaw-plugins/daedalus-agent-mqtt-channel/index.test.ts create mode 100644 extensions/openclaw-plugins/daedalus-agent-mqtt-channel/src/model-usage.test.ts create mode 100644 extensions/openclaw-plugins/daedalus-agent-mqtt-channel/src/model-usage.ts diff --git a/doc/system-feature-list.md b/doc/system-feature-list.md index 537a524..5d20688 100644 --- a/doc/system-feature-list.md +++ b/doc/system-feature-list.md @@ -33,7 +33,7 @@ ### 2.2 总览、同步与版本 -- [x][T] 仪表盘展示数字员工、模板、Skill、业务系统数量和 Token 用量等统计；用量表（用户）与用量表（用户维度）均按今日 Token 用量倒序、每页 10 条，仅展示今日有消耗的 Agent/用户，并直接聚合 Model Gateway 账本。 +- [x][T] 仪表盘展示数字员工、模板、Skill、业务系统数量和 Token 用量等统计；“用量表（数字员工）”与“用量表（用户）”均按今日 Token 用量倒序、每页 10 条，仅展示今日有消耗的 Agent/用户，并直接聚合 Model Gateway 账本。 - [x][T] 启动同步等待 `service.ping`，随后执行 `version.list`、`catalog.snapshot`；目录快照成功后延迟从 Model Gateway 用量账本刷新 Agent 快照。 - [x][T] 目录和用量按计划自动刷新；每次完成后按配置间隔再次执行；页面可轮询同步状态。 - [x][T] “同步数据”手动执行目录快照再执行用量刷新，并展示各对象的状态、开始/结束时间、错误和摘要。 @@ -122,7 +122,7 @@ - [x][T] 管理 Web 提供面向 `/api/external/*` 对外接口的 OpenAPI 3.0 文档和本地 Swagger UI：文档入口为 `/doc`，规范 JSON 为 `/doc/openapi.json`，包含应用场景、鉴权、参数用途/限制/可选项、请求体和常见错误说明；基础设置的版本信息下方提供“API 文档”入口。 - [x][T] 持久外部会话 ID 使用 `s-` 加 30 位数字格式；同一用户-Agent 复用会话；会话创建校验 Agent 状态和用户授权。同步调用使用独立的一次性 MQTT 关联 ID，不属于持久外部会话。 - [x][T] 外部调用在每次解析业务系统时重新校验会话有效性、Agent 状态、业务系统状态和 Agent 业务系统策略。 -- [x][T] 内部 CUI API：查询可用 Agent 目录；内部 invoke API：解析授权业务系统、获取业务系统 base URL、记录调用日志；Model Gateway 用量内部 API 写入幂等事件。 +- [x][T] 内部 CUI API：查询可用 Agent 目录；内部 invoke API：解析授权业务系统、获取业务系统 base URL、记录调用日志；Model Gateway 用量内部 API 写入幂等事件并提供每日配额检查；OpenClaw MQTT 插件通过 `llm_output` 落账并在 `before_agent_run` 阻断超限请求，提示明确指出超限用户或数字员工。 - [x][T] 管理员用量 API：`GET /api/usage/summary` 支持按 Agent、senderId、Agent+senderId、模型和时间窗口聚合；`GET /api/usage/agents/{agentId}/senders` 查询指定 Agent 的 senderId 明细。 - [x][T] API 鉴权分层：管理端登录 Token、服务 Bearer Token、内部请求校验；localhost 请求按配置可走内部通道。 - [x][T] 外部 Cookie 缓存并按 `hzg` Provider 获取；业务系统调用结果写入统一调用日志和 Payload 制品。 @@ -171,7 +171,7 @@ | 管理 Web 服务 | 路由、鉴权、用户/Agent/模板/Skill/LLM/MCP/业务系统 CRUD、内置模板编辑/重命名与删除保护、模板主/备选模型和 Agent 继承、Agent 用户访问策略（全部用户模式允许目录外未知用户名、指定用户模式仍需显式授权）、用户每日配额维护及今日用量/状态、用户改名与网关事件事务更新、Agent 多实例 MCP 选定服务同步及普通/沙箱 deny 投影、外部 Agent 目录匿名公开筛选与 `is_public` 标记、同步 Agent HTTP 接口（参数校验、服务鉴权、可选 Cookie、首次同步调用自动创建非管理员 Agent 用户、用户/Agent 任一配额超限时明确拒绝、临时会话业务系统调用、MQTT final-only 行为、临时 session 清理、成功后刷新用量并计入每日配额）、管理端 Agent 对话测试配额拦截、Model Gateway 用量事件幂等写入、按 Agent/senderId 聚合查询、Agent sender 明细、运行凭据密码重置及容器重启请求、OpenAPI/Swagger 文档外部接口和用量接口、审计、同步、健康检查、日志下载、SQLite 回滚和制品安全 | `packages/daedalus-agent-management-web/server/test/app-smoke.test.js`、`packages/daedalus-agent-management-web/server/test/model-gateway-usage-service.test.js`、`packages/daedalus-agent-management-web/server/test/model-gateway-usage-route.test.js`、`packages/daedalus-agent-management-web/server/test/catalog-sync-service.test.js`、`packages/daedalus-agent-management-web/server/test/db-reset-migration.test.js`、`packages/daedalus-agent-management-web/server/test/user-quota.test.js`、`packages/daedalus-agent-management-web/server/test/portal-service-alignment.test.js`、`packages/daedalus-agent-management-web/server/test/runtime-env-config-service.test.js`、`packages/daedalus-agent-management-web/server/test/runtime-settings-route.test.js`、`packages/daedalus-agent-management-web/server/test/topic-ping-service.test.js`、`packages/daedalus-agent-management-web/server/test/agent-user-policy.test.js`、`packages/daedalus-agent-management-web/server/test/external-service.test.js`、`packages/daedalus-agent-management-web/server/test/agent-service-alignment.test.js`、`packages/daedalus-agent-management-web/server/test/env-config.test.js`、`packages/daedalus-agent-management-web/server/test/mcp-service.test.js`、`packages/daedalus-agent-management-web/server/management-test/mcp-sync.test.ts`、其他 `server/test/`、`server/management-test/` | | 管理 Web 前端 | API 客户端鉴权/超时/错误归一化、同步门控、重启轮询、格式化、浏览器覆盖契约、员工与模板列表操作精简、内置模板编辑、名称/编辑详情入口、详情 tabs 保存流程、员工概览主模型/备选模型/每日配额快捷修改及打开状态下的 Token 用量同步刷新、用户每日限额/今日用量/状态列表、用户新增/编辑/删除、Dashboard 用户/Agent 用量表 10 条分页与零用量过滤、员工技能/业务系统/MCP 与模板设置布局一致及内置 Skill 隐藏、编辑页不展示同步调用 URL、模板文本文件切换后的编辑器可见性、设置页 API 文档入口、运行凭据重置按钮/确认框及“刷新状态”按钮 | `packages/daedalus-agent-management-web/src/**/*.test.js`、`packages/daedalus-agent-management-web/src/pages/AgentsPage.test.jsx`、`packages/daedalus-agent-management-web/src/pages/AgentUsersPage.test.jsx`、`packages/daedalus-agent-management-web/src/pages/DashboardPage.test.jsx`、`packages/daedalus-agent-management-web/test/browser-coverage.test.js`、`packages/daedalus-agent-management-web/test/run-browser-e2e.js`、`packages/daedalus-agent-management-web/test/integration/website-full.js` | | invoke CLI | 参数、JSON/body/query 来源、Binding schema、socket 协议、超时、大小限制、Provider 错误和调用日志 | `packages/daedalus-agent-apps-invoke-cli/test/` | -| MQTT 插件 | 账号配置、旧配置兼容、环境变量覆盖、连接/断线重连、通配符、入站路由、普通消息串行队列、停止与 steer/tell 控制队列、active-run 非阻塞 steer、运行时接收 Promise 未完成时的确认与原 final 投递、原始会话目标、OpenClaw deliver 的 text/kind 无加工透传、出站 envelope、日志 | `extensions/openclaw-plugins/daedalus-agent-mqtt-channel/src/*test.ts` | +| MQTT 插件 | 账号配置、旧配置兼容、环境变量覆盖、连接/断线重连、通配符、入站路由、普通消息串行队列、停止与 steer/tell 控制队列、active-run 非阻塞 steer、运行时接收 Promise 未完成时的确认与原 final 投递、原始会话目标、OpenClaw deliver 的 text/kind 无加工透传、出站 envelope、日志、Model Gateway `llm_output` 用量落账、`before_agent_run` 配额阻断和待写入同步 | `extensions/openclaw-plugins/daedalus-agent-mqtt-channel/src/*test.ts`、`extensions/openclaw-plugins/daedalus-agent-mqtt-channel/index.test.ts` | | Docker 集成 | 登录、版本/同步、Provider/Model/模板/MCP/Agent 生命周期、模板继承、工作区/Skill、Ontology、MQTT `/stop`/`/steer`/`/tell` 的 final 确认及普通队列保留、重启、启动钩子、包持久化、文件输出、会话隔离和压测 | `test/case/integration/`、`test/suite/integration/`、`test/case/stress/` | | Docker 升级与品牌契约 | OpenClaw/Node 版本、插件 seed、迁移、重启保护、持久化 MQTT 账号密码与运行凭据同步、DA_ 环境变量前缀和视觉品牌 | `test/case/upgrade/openclaw-upgrade-contract.test.js`、`test/case/upgrade/`、`test/case/branding/` | diff --git a/docker/assets/config.json b/docker/assets/config.json index c074978..64efdd4 100644 --- a/docker/assets/config.json +++ b/docker/assets/config.json @@ -41,7 +41,10 @@ }, "entries": { "daedalus-agent-mqtt-channel": { - "enabled": true + "enabled": true, + "hooks": { + "allowConversationAccess": true + } } } }, diff --git a/docker/assets/scripts/kernel-startup.sh b/docker/assets/scripts/kernel-startup.sh index b788a59..504a4c5 100644 --- a/docker/assets/scripts/kernel-startup.sh +++ b/docker/assets/scripts/kernel-startup.sh @@ -2274,6 +2274,7 @@ patch_openclaw_config() { | .plugins.allow = ((.plugins.allow // []) + ["daedalus-agent-mqtt-channel"] | unique) | .plugins.load.paths = ((.plugins.load.paths // []) + [$seedMqttPluginPath] | unique) | .plugins.entries["daedalus-agent-mqtt-channel"].enabled = true + | .plugins.entries["daedalus-agent-mqtt-channel"].hooks.allowConversationAccess = true | if ((.channels["daedalus-agent-mqtt-channel"].accounts? | type) == "object") and ($mqttPassword != "") then .channels["daedalus-agent-mqtt-channel"].accounts |= with_entries( .value.username = (.value.username // "daedalus-agent") diff --git a/extensions/openclaw-plugins/daedalus-agent-mqtt-channel/DEV.md b/extensions/openclaw-plugins/daedalus-agent-mqtt-channel/DEV.md index ade18fc..187e3eb 100644 --- a/extensions/openclaw-plugins/daedalus-agent-mqtt-channel/DEV.md +++ b/extensions/openclaw-plugins/daedalus-agent-mqtt-channel/DEV.md @@ -30,6 +30,8 @@ setup wizard 配置写入逻辑。 - `src/runtime.ts` OpenClaw runtime 注入与读取。 +- `src/model-usage.ts` + Model Gateway 用量事件落账、用户/数字员工配额检查，以及异步落账与下一轮运行之间的同步。 - `src/__mocks__/mqtt.ts` 测试用 MQTT mock。 - `src/*.test.ts` @@ -79,6 +81,8 @@ MQTT 客户端测试。 - `src/channel.test.ts` 插件收发、路由、会话和账号行为测试。 +- `src/model-usage.test.ts`、`index.test.ts` + Model Gateway Hook 映射、幂等、配额阻断、待写入同步和插件 Hook 注册测试。 ## 构建与发布 diff --git a/extensions/openclaw-plugins/daedalus-agent-mqtt-channel/README.md b/extensions/openclaw-plugins/daedalus-agent-mqtt-channel/README.md index 0d8a7d5..f17e5f6 100644 --- a/extensions/openclaw-plugins/daedalus-agent-mqtt-channel/README.md +++ b/extensions/openclaw-plugins/daedalus-agent-mqtt-channel/README.md @@ -91,7 +91,10 @@ openclaw plugins update daedalus-agent-mqtt-channel "allow": ["daedalus-agent-mqtt-channel"], "entries": { "daedalus-agent-mqtt-channel": { - "enabled": true + "enabled": true, + "hooks": { + "allowConversationAccess": true + } } } } @@ -174,6 +177,17 @@ openclaw plugins update daedalus-agent-mqtt-channel | `MQTT_CLIENT_ID` | `clientId` | 共享物理连接的客户端 ID | | `MQTT_CA_PATH` | `tls.ca` | 自定义 CA 路径字段；当前不会读取证书文件，因此暂不产生实际 TLS 效果 | +### Model Gateway 用量与配额 + +插件在模型输出 Hook 中把 Provider Token 用量写入管理端 Model Gateway 账本，并在每次 Agent 运行前检查用户和数字员工的每日配额。非内置插件使用这些对话 Hook 时，必须在 `plugins.entries["daedalus-agent-mqtt-channel"].hooks` 中开启 `allowConversationAccess`。 + +| 环境变量 | 默认值 | 含义和限制 | +| --- | --- | --- | +| `DA_MANAGEMENT_MODEL_GATEWAY_URL` | `http://127.0.0.1:${DA_MANAGEMENT_CONSOLE_PORT}/api/internal/model-gateway` | 管理端 Model Gateway 内部 API 地址 | +| `DA_MANAGEMENT_INTERNAL_TOKEN` | 空 | 管理端内部 API Bearer Token；同一容器回环访问通常无需设置 | + +用量事件按 `agentId`、`senderId`、Provider 和 Model 记录；账本写入失败不会修改已经完成的模型响应，但后续运行的配额检查会在管理端不可用时阻断请求。 + 也可以通过交互式命令配置 `default` 账号： ```bash diff --git a/extensions/openclaw-plugins/daedalus-agent-mqtt-channel/index.test.ts b/extensions/openclaw-plugins/daedalus-agent-mqtt-channel/index.test.ts new file mode 100644 index 0000000..3a2d13a --- /dev/null +++ b/extensions/openclaw-plugins/daedalus-agent-mqtt-channel/index.test.ts @@ -0,0 +1,18 @@ +import plugin from "./index.js"; + +it("registers model usage hooks in the full OpenClaw runtime", () => { + const on = jasmine.createSpy("on"); + const api = { + id: "daedalus-agent-mqtt-channel", + registrationMode: "full", + registerChannel: jasmine.createSpy("registerChannel"), + on, + runtime: {}, + } as any; + + plugin.register(api); + + expect(on).toHaveBeenCalledWith("model_call_started", jasmine.any(Function)); + expect(on).toHaveBeenCalledWith("llm_output", jasmine.any(Function)); + expect(on).toHaveBeenCalledWith("before_agent_run", jasmine.any(Function)); +}); diff --git a/extensions/openclaw-plugins/daedalus-agent-mqtt-channel/index.ts b/extensions/openclaw-plugins/daedalus-agent-mqtt-channel/index.ts index 6c3b6c5..525e49c 100644 --- a/extensions/openclaw-plugins/daedalus-agent-mqtt-channel/index.ts +++ b/extensions/openclaw-plugins/daedalus-agent-mqtt-channel/index.ts @@ -3,6 +3,11 @@ import { defineChannelPluginEntry } from "openclaw/plugin-sdk/channel-core"; import { mqttPlugin } from "./src/channel.js"; import { setMqttRuntime } from "./src/runtime.js"; import { mqttChannelConfigJsonSchema } from "./src/config-schema.js"; +import { + createModelUsageHooks, + defaultModelUsageEndpoint, + defaultModelUsageToken, +} from "./src/model-usage.js"; const plugin = defineChannelPluginEntry({ id: "daedalus-agent-mqtt-channel", @@ -11,6 +16,17 @@ const plugin = defineChannelPluginEntry({ plugin: mqttPlugin, configSchema: mqttChannelConfigJsonSchema, setRuntime: setMqttRuntime, + registerFull(api) { + const hooks = createModelUsageHooks({ + endpoint: defaultModelUsageEndpoint(), + token: defaultModelUsageToken(), + logger: api.logger, + runtimeId: process.env.HOSTNAME || "openclaw", + }); + api.on("model_call_started", hooks.model_call_started); + api.on("llm_output", hooks.llm_output); + api.on("before_agent_run", hooks.before_agent_run); + }, }); export default plugin; diff --git a/extensions/openclaw-plugins/daedalus-agent-mqtt-channel/src/model-usage.test.ts b/extensions/openclaw-plugins/daedalus-agent-mqtt-channel/src/model-usage.test.ts new file mode 100644 index 0000000..0520998 --- /dev/null +++ b/extensions/openclaw-plugins/daedalus-agent-mqtt-channel/src/model-usage.test.ts @@ -0,0 +1,159 @@ +import { + buildModelUsageEventPayload, + createModelUsageHooks, + quotaBlockResult, +} from "./model-usage.js"; + +describe("model usage gateway hooks", () => { + it("maps OpenClaw output and hook context to an idempotent gateway event", () => { + expect(buildModelUsageEventPayload( + { + runId: "run-1", + sessionId: "session-1", + provider: "openai", + model: "gpt-5", + resolvedRef: "openai/gpt-5", + usage: { input: 12, output: 8, cacheRead: 3, cacheWrite: 1, total: 24 }, + }, + { + agentId: "finance-agent", + senderId: "alice", + sessionKey: "agent:finance-agent:mqtt:alice", + channel: "daedalus-agent-mqtt-channel", + }, + { runtimeId: "runtime-1", requestedModelRef: "corp/gpt-5", callId: "call-1" }, + )).toEqual({ + eventId: "run-1:call-1", + requestId: "run-1:call-1", + traceId: "run-1", + runtimeId: "runtime-1", + agentId: "finance-agent", + senderId: "alice", + sessionId: "session-1", + channel: "daedalus-agent-mqtt-channel", + requestedModelRef: "corp/gpt-5", + actualProviderId: "openai", + actualModelRef: "openai/gpt-5", + promptTokens: 12, + completionTokens: 8, + cachedTokens: 4, + totalTokens: 24, + usageSource: "provider", + billable: true, + success: true, + }); + }); + + it("blocks with the subject when the management quota check reports a violation", async () => { + const fetchImpl = jasmine.createSpy("fetch").and.resolveTo(new Response(JSON.stringify({ + allowed: false, + violations: [{ subject: "user", name: "alice", current: 100, limit: 100 }], + }), { status: 200, headers: { "content-type": "application/json" } })); + const hooks = createModelUsageHooks({ + endpoint: "http://127.0.0.1:3030/api/internal/model-gateway", + fetchImpl: fetchImpl as any, + logger: { warn() {}, error() {} }, + }); + + const result = await hooks.before_agent_run( + { prompt: "hello", messages: [], senderId: "alice" }, + { agentId: "finance-agent", senderId: "alice", sessionId: "session-1" }, + ); + + expect(result).toEqual(quotaBlockResult("用户 alice 已超出每日用量限额（当前 100，限额 100），请等待配额重置")); + expect(fetchImpl).toHaveBeenCalledWith( + "http://127.0.0.1:3030/api/internal/model-gateway/quota-check", + jasmine.objectContaining({ method: "POST" }), + ); + }); + + it("does not make a completed model response fail when usage recording is unavailable", async () => { + const fetchImpl = jasmine.createSpy("fetch").and.rejectWith(new Error("management unavailable")); + const hooks = createModelUsageHooks({ + endpoint: "http://127.0.0.1:3030/api/internal/model-gateway", + fetchImpl: fetchImpl as any, + logger: { warn() {}, error() {} }, + }); + + await expectAsync(hooks.llm_output( + { + runId: "run-1", + sessionId: "session-1", + provider: "openai", + model: "gpt-5", + usage: { input: 1, output: 2, total: 3 }, + }, + { agentId: "finance-agent", senderId: "alice" }, + )).toBeResolved(); + }); + + it("uses the same request id when OpenClaw retries an identical output hook", async () => { + const responses = [ + new Response("{}", { status: 201 }), + new Response("{}", { status: 200 }), + ]; + const fetchImpl = jasmine.createSpy("fetch").and.callFake(async () => responses.shift()!); + const hooks = createModelUsageHooks({ + endpoint: "http://127.0.0.1:3030/api/internal/model-gateway", + fetchImpl: fetchImpl as any, + logger: { warn() {}, error() {} }, + }); + const event = { + runId: "run-retry", + sessionId: "session-1", + provider: "openai", + model: "gpt-5", + usage: { input: 1, output: 2, total: 3 }, + }; + + await hooks.llm_output(event, { agentId: "finance-agent", senderId: "alice" }); + await hooks.llm_output(event, { agentId: "finance-agent", senderId: "alice" }); + + const first = JSON.parse(fetchImpl.calls.argsFor(0)[1].body); + const second = JSON.parse(fetchImpl.calls.argsFor(1)[1].body); + expect(first.requestId).toBe(second.requestId); + }); + + it("waits for pending usage recording before checking the next run quota", async () => { + let releaseUsage: ((response: Response) => void) | undefined; + const pendingUsage = new Promise((resolve) => { + releaseUsage = resolve; + }); + const fetchImpl = jasmine.createSpy("fetch").and.callFake((url: string) => { + if (url.endsWith("/usage-events")) { + return pendingUsage; + } + return Promise.resolve(new Response(JSON.stringify({ allowed: false, violations: [ + { subject: "agent", name: "finance-agent", current: 3, limit: 3 }, + ] }), { status: 200, headers: { "content-type": "application/json" } })); + }); + const hooks = createModelUsageHooks({ + endpoint: "http://127.0.0.1:3030/api/internal/model-gateway", + fetchImpl: fetchImpl as any, + logger: { warn() {}, error() {} }, + }); + + const usagePromise = hooks.llm_output({ + runId: "run-before-next", + sessionId: "session-1", + provider: "openai", + model: "gpt-5", + usage: { input: 1, output: 2, total: 3 }, + }, { agentId: "finance-agent", senderId: "alice" }); + await Promise.resolve(); + + const quotaPromise = hooks.before_agent_run( + { prompt: "next", messages: [], senderId: "alice" }, + { agentId: "finance-agent", senderId: "alice" }, + ); + await Promise.resolve(); + expect(fetchImpl.calls.count()).toBe(1); + + releaseUsage?.(new Response("{}", { status: 201 })); + await usagePromise; + await expectAsync(quotaPromise).toBeResolvedTo( + quotaBlockResult("数字员工 finance-agent 已超出每日用量限额（当前 3，限额 3），请等待配额重置"), + ); + expect(fetchImpl.calls.count()).toBe(2); + }); +}); diff --git a/extensions/openclaw-plugins/daedalus-agent-mqtt-channel/src/model-usage.ts b/extensions/openclaw-plugins/daedalus-agent-mqtt-channel/src/model-usage.ts new file mode 100644 index 0000000..5de78f9 --- /dev/null +++ b/extensions/openclaw-plugins/daedalus-agent-mqtt-channel/src/model-usage.ts @@ -0,0 +1,283 @@ +type UsageEvent = { + runId?: string; + callId?: string; + sessionId?: string; + provider?: string; + model?: string; + resolvedRef?: string; + usage?: { + input?: number; + output?: number; + cacheRead?: number; + cacheWrite?: number; + total?: number; + }; +}; + +type HookContext = { + agentId?: string; + senderId?: string; + requesterSenderId?: string; + senderExternalId?: string; + chatId?: string; + sessionId?: string; + sessionKey?: string; + channel?: string; + channelId?: string; + modelProviderId?: string; + modelId?: string; + channelContext?: { + sender?: { id?: string }; + chat?: { id?: string }; + }; +}; + +type ModelUsageLogger = { + warn?: (message: string) => void; + error?: (message: string) => void; +}; + +type FetchLike = (input: string | URL, init?: RequestInit) => Promise; +type QuotaHookResult = { + outcome: "block"; + reason: string; + message?: string; + category?: string; +}; + +const DEFAULT_ENDPOINT = "http://127.0.0.1:3030/api/internal/model-gateway"; +const DEFAULT_TIMEOUT_MS = 2500; + +function text(value: unknown): string { + return String(value ?? "").trim(); +} + +function token(value: unknown): number { + const parsed = Number(value); + return Number.isSafeInteger(parsed) && parsed >= 0 ? parsed : 0; +} + +function endpointPath(endpoint: string, suffix: string): string { + return `${endpoint.replace(/\/+$/, "")}/${suffix}`; +} + +function resolveSender(context: HookContext, event?: { senderId?: string }): string { + return text( + event?.senderId + || context.senderId + || context.requesterSenderId + || context.channelContext?.sender?.id + || context.senderExternalId + || context.chatId + || context.channelContext?.chat?.id, + ); +} + +function resolveAgent(context: HookContext): string { + return text(context.agentId); +} + +function usageErrorMessage(): string { + return "每日配额校验服务不可用，请稍后重试"; +} + +export function buildModelUsageEventPayload( + event: UsageEvent, + context: HookContext, + options: { runtimeId?: string; requestedModelRef?: string; callId?: string } = {}, +): Record { + const runId = text(event.runId); + const callId = text(options.callId || event.callId); + const requestId = callId ? `${runId}:${callId}` : runId; + const usage = event.usage || {}; + const promptTokens = token(usage.input); + const completionTokens = token(usage.output); + const cachedTokens = token(usage.cacheRead) + token(usage.cacheWrite); + const totalTokens = token(usage.total || promptTokens + completionTokens); + const provider = text(event.provider || context.modelProviderId); + const model = text(event.resolvedRef || (provider && event.model ? `${provider}/${event.model}` : event.model)); + const agentId = resolveAgent(context); + const senderId = resolveSender(context); + + return { + eventId: requestId, + requestId, + traceId: runId, + runtimeId: text(options.runtimeId || process.env.HOSTNAME || "openclaw"), + agentId, + senderId, + sessionId: text(event.sessionId || context.sessionId || context.sessionKey), + channel: text(context.channel || context.channelId || "daedalus-agent-mqtt-channel"), + requestedModelRef: text(options.requestedModelRef || context.modelId || model), + actualProviderId: provider, + actualModelRef: model, + promptTokens, + completionTokens, + cachedTokens, + totalTokens, + usageSource: "provider", + billable: true, + success: true, + }; +} + +export function quotaBlockResult(message: string): QuotaHookResult { + return { + outcome: "block", + reason: "daily quota exceeded", + message, + category: "cost_limit", + }; +} + +function formatViolation(violation: any): string { + const subject = violation?.subject === "agent" ? "数字员工" : "用户"; + const name = text(violation?.name) || (subject === "用户" ? "当前用户" : "当前数字员工"); + const current = token(violation?.current); + const limit = token(violation?.limit); + return `${subject} ${name} 已超出每日用量限额（当前 ${current}，限额 ${limit}），请等待配额重置`; +} + +function blockFromQuotaResponse(payload: any): QuotaHookResult | undefined { + if (payload?.allowed !== false) { + return undefined; + } + const violations = Array.isArray(payload.violations) ? payload.violations : []; + const message = violations.length > 0 + ? violations.map(formatViolation).join("；") + : "每日用量限额已超出，请等待配额重置"; + return quotaBlockResult(message); +} + +export function createModelUsageHooks({ + endpoint = DEFAULT_ENDPOINT, + token: authToken = "", + fetchImpl = globalThis.fetch, + logger = console, + timeoutMs = DEFAULT_TIMEOUT_MS, + runtimeId = process.env.HOSTNAME || "openclaw", +}: { + endpoint?: string; + token?: string; + fetchImpl?: FetchLike; + logger?: ModelUsageLogger; + timeoutMs?: number; + runtimeId?: string; +} = {}) { + const callQueues = new Map(); + const lastOutputByRun = new Map(); + const pendingUsageWrites = new Set>(); + let sequence = 0; + + const request = async (suffix: string, payload: Record): Promise => { + if (typeof fetchImpl !== "function") { + throw new Error("fetch is unavailable"); + } + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), timeoutMs); + try { + const response = await fetchImpl(endpointPath(endpoint, suffix), { + method: "POST", + headers: { + "content-type": "application/json", + ...(authToken ? { authorization: `Bearer ${authToken}` } : {}), + }, + body: JSON.stringify(payload), + signal: controller.signal, + }); + const body = await response.json().catch(() => ({})); + if (!response.ok) { + throw new Error(`management gateway returned ${response.status}`); + } + return body; + } finally { + clearTimeout(timer); + } + }; + + return { + model_call_started(event: { runId?: string; callId?: string; context?: HookContext }) { + const source = event.context || event; + const runId = text(event.runId || (source as HookContext).sessionKey); + const callId = text(event.callId); + if (!runId || !callId) return; + const queue = callQueues.get(runId) || []; + queue.push(callId); + callQueues.set(runId, queue); + }, + + async before_agent_run( + event: { senderId?: string; context?: HookContext }, + suppliedContext?: HookContext, + ): Promise { + const context = suppliedContext || event.context || {}; + const agentId = resolveAgent(context); + const senderId = resolveSender(context, event); + if (!agentId || !senderId) { + const message = "无法识别用户或数字员工，暂不可调用"; + logger.warn?.(message); + return quotaBlockResult(message); + } + + try { + await Promise.all([...pendingUsageWrites]); + const result = await request("quota-check", { agentId, senderId }); + return blockFromQuotaResponse(result); + } catch (error) { + logger.error?.(`daily quota check failed: ${String(error)}`); + return quotaBlockResult(usageErrorMessage()); + } + }, + + async llm_output(event: UsageEvent & { context?: HookContext }, suppliedContext?: HookContext) { + const context = suppliedContext || event.context || {}; + if (!event.usage) return; + const runId = text(event.runId); + const queue = callQueues.get(runId) || []; + const signature = JSON.stringify([ + text(event.provider), + text(event.model), + event.usage || {}, + ]); + const previous = lastOutputByRun.get(runId); + const callId = text(event.callId || queue.shift()) + || (previous?.signature === signature ? previous.callId : "") + || `call-${++sequence}`; + if (runId) { + lastOutputByRun.set(runId, { signature, callId }); + if (lastOutputByRun.size > 2048) { + const oldest = lastOutputByRun.keys().next().value; + if (oldest) lastOutputByRun.delete(oldest); + } + } + if (queue.length > 0) callQueues.set(runId, queue); + else callQueues.delete(runId); + const payload = buildModelUsageEventPayload(event, context, { runtimeId, callId }); + if (!payload.agentId || !payload.senderId) { + logger.warn?.("skip model usage event without agentId or senderId"); + return; + } + const writePromise = request("usage-events", payload) + .catch((error) => { + logger.warn?.(`model usage event recording failed: ${String(error)}`); + }); + pendingUsageWrites.add(writePromise); + try { + await writePromise; + } finally { + pendingUsageWrites.delete(writePromise); + } + }, + }; +} + +export function defaultModelUsageEndpoint(): string { + const explicit = text(process.env.DA_MANAGEMENT_MODEL_GATEWAY_URL); + if (explicit) return explicit; + const port = text(process.env.DA_MANAGEMENT_CONSOLE_PORT) || "3030"; + return `http://127.0.0.1:${port}/api/internal/model-gateway`; +} + +export function defaultModelUsageToken(): string { + return text(process.env.DA_MANAGEMENT_INTERNAL_TOKEN); +} diff --git a/extensions/openclaw-plugins/daedalus-agent-mqtt-channel/test/run-jasmine.ts b/extensions/openclaw-plugins/daedalus-agent-mqtt-channel/test/run-jasmine.ts index ac510d1..616b9d0 100644 --- a/extensions/openclaw-plugins/daedalus-agent-mqtt-channel/test/run-jasmine.ts +++ b/extensions/openclaw-plugins/daedalus-agent-mqtt-channel/test/run-jasmine.ts @@ -5,7 +5,7 @@ async function main(): Promise { runner.exitOnCompletion = false; runner.loadConfig({ spec_dir: ".", - spec_files: ["src/**/*.test.ts"], + spec_files: ["src/**/*.test.ts", "*.test.ts"], helpers: ["test/setup.ts"], random: false, stopSpecOnExpectationFailure: false, diff --git a/packages/daedalus-agent-management-web/docs/spec.md b/packages/daedalus-agent-management-web/docs/spec.md index d15f725..94a77d3 100644 --- a/packages/daedalus-agent-management-web/docs/spec.md +++ b/packages/daedalus-agent-management-web/docs/spec.md @@ -102,7 +102,7 @@ Provider 表示 OpenAI 兼容模型服务的 `baseUrl + SK`。Model 归属 Provi ### 总览用量表 -总览页面提供“用量表（用户）”和“用量表（用户维度）”两张表，均直接按中国自然日从 Model Gateway 账本聚合，按今日用量倒序、每页 10 条，仅展示今日有消耗的数字员工或用户。 +总览页面提供“用量表（数字员工）”和“用量表（用户）”两张表，均直接按中国自然日从 Model Gateway 账本聚合，按今日用量倒序、每页 10 条，仅展示今日有消耗的数字员工或用户。 ## 设置页 diff --git a/packages/daedalus-agent-management-web/server/src/api/routes/index.js b/packages/daedalus-agent-management-web/server/src/api/routes/index.js index e224788..f50126b 100644 --- a/packages/daedalus-agent-management-web/server/src/api/routes/index.js +++ b/packages/daedalus-agent-management-web/server/src/api/routes/index.js @@ -359,6 +359,17 @@ export function createRoutes(services) { res.status(result.inserted ? 201 : 200).json(result); })); + router.post("/internal/model-gateway/quota-check", requireInternal, asyncRoute(async (req, res) => { + const externalService = services.externalService; + if (!externalService || typeof externalService.checkModelGatewayQuota !== "function") { + throw serviceUnavailable("Model Gateway quota service is unavailable"); + } + res.json(externalService.checkModelGatewayQuota({ + agentId: req.body?.agentId, + senderId: req.body?.senderId + })); + })); + router.get("/external/cookie", requireServiceApiAuth, asyncRoute(async (req, res) => { const result = await services.externalService.getCookie( req.query.sessionId, diff --git a/packages/daedalus-agent-management-web/server/src/docs/openapi.js b/packages/daedalus-agent-management-web/server/src/docs/openapi.js index 84664f0..60dad93 100644 --- a/packages/daedalus-agent-management-web/server/src/docs/openapi.js +++ b/packages/daedalus-agent-management-web/server/src/docs/openapi.js @@ -188,6 +188,10 @@ const modelGatewayUsageEventBody = jsonBody("Model Gateway 用量事件。事件 latencyMs: integerField("模型请求耗时，单位毫秒。", { minimum: 0 }), createdAt: textField("事件发生时间；支持 ISO-8601，默认当前时间。", { format: "date-time" }) }, ["agentId", "senderId"]); +const modelGatewayQuotaCheckBody = jsonBody("Model Gateway 每日配额检查请求。", { + agentId: textField("待调用的数字员工 slug。", { minLength: 1 }), + senderId: textField("发起调用的 senderId。", { minLength: 1 }) +}, ["agentId", "senderId"]); const usageSummarySchema = { type: "object", properties: { @@ -306,6 +310,11 @@ const ROUTES = [ description: "事件已存在，未重复计量。" }] }), + route("post", "/internal/model-gateway/quota-check", "Model Gateway 用量", "检查每日配额", "OpenClaw 在开始 Agent 运行前调用，检查指定用户和数字员工的中国自然日用量；任一超限都会返回明确主体。", { + authDescription: "需要内部请求校验；非本机请求必须携带服务 Bearer Token。", + body: modelGatewayQuotaCheckBody, + successSchema: { $ref: "#/components/schemas/ModelGatewayQuotaCheckResult" } + }), route("get", "/usage/summary", "Model Gateway 用量", "查询模型用量统计", "管理员按日期窗口和 Agent、senderId、模型或时间维度查询 Model Gateway 计量结果。", { authDescription: "需要管理端管理员会话 Token。", query: [ @@ -847,6 +856,20 @@ export const OPENAPI_DOCUMENT = { eventId: textField("已接收的事件 ID。"), inserted: booleanField("本次是否新写入；重复事件为 false。") }, "Model Gateway 用量事件写入结果。"), + ModelGatewayQuotaCheckResult: objectSchema({ + allowed: booleanField("是否允许开始本次 Agent 运行。"), + violations: { + type: "array", + description: "当前已超限的用户或数字员工；为空表示允许。", + items: objectSchema({ + subject: textField("超限主体类型。", { enum: ["user", "agent"] }), + name: textField("超限主体名称。"), + current: integerField("今日用量。", { minimum: 0 }), + limit: integerField("每日限额。", { minimum: 0 }), + period: textField("配额周期。") + }) + } + }, "Model Gateway 每日配额检查结果。"), Pagination: objectSchema({ page: integerField("当前页码。", { minimum: 1 }), pageSize: integerField("每页条数。", { minimum: 1 }), diff --git a/packages/daedalus-agent-management-web/server/src/services/external-service.js b/packages/daedalus-agent-management-web/server/src/services/external-service.js index c455e2c..0e2e644 100644 --- a/packages/daedalus-agent-management-web/server/src/services/external-service.js +++ b/packages/daedalus-agent-management-web/server/src/services/external-service.js @@ -110,6 +110,47 @@ export class ExternalService { return findTodayQuotaViolation(this.db, user, { senderId: user?.username }); } + checkModelGatewayQuota({ agentId, senderId } = {}) { + const normalizedAgentId = String(agentId || "").trim(); + const normalizedSenderId = String(senderId || "").trim(); + if (!normalizedAgentId || !normalizedSenderId) { + throw badRequest("agentId 和 senderId 不能为空"); + } + + const agent = this.db.prepare(` + SELECT id, slug, daily_limit + FROM agents + WHERE slug = ? + `).get(normalizedAgentId); + if (!agent) { + throw notFound(`数字员工不存在：${normalizedAgentId}`); + } + + const user = this.loadAgentUserByUsername(normalizedSenderId); + const violations = []; + const agentViolation = this.getAgentQuotaViolation(agent); + if (agentViolation) { + violations.push({ + subject: "agent", + name: agent.slug, + current: agentViolation.current, + limit: agentViolation.limit, + period: agentViolation.period + }); + } + const userViolation = user ? this.getUserQuotaViolation(user) : null; + if (userViolation) { + violations.push({ + subject: "user", + name: user.username, + current: userViolation.current, + limit: userViolation.limit, + period: userViolation.period + }); + } + return { allowed: violations.length === 0, violations }; + } + loadAgentUserById(agentUserId) { try { return this.db.prepare(` diff --git a/packages/daedalus-agent-management-web/server/src/services/portal-service.js b/packages/daedalus-agent-management-web/server/src/services/portal-service.js index 385f820..7f71a2b 100644 --- a/packages/daedalus-agent-management-web/server/src/services/portal-service.js +++ b/packages/daedalus-agent-management-web/server/src/services/portal-service.js @@ -1529,7 +1529,7 @@ export class PortalService { templates: templateCount, skills: skillCount, systems: systemCount, - today_tokens: usageStatsReady ? totalTokenUsage : null + today_tokens: usageLedgerReady || usageStatsReady ? totalTokenUsage : null }, recentInvocations, agentUsage, diff --git a/packages/daedalus-agent-management-web/server/test/app-smoke.test.js b/packages/daedalus-agent-management-web/server/test/app-smoke.test.js index 8f8a5a6..daf1441 100644 --- a/packages/daedalus-agent-management-web/server/test/app-smoke.test.js +++ b/packages/daedalus-agent-management-web/server/test/app-smoke.test.js @@ -43,7 +43,7 @@ it("assembles the management server and maps application and parser errors", asy expect(docsJson.openapi).toBe("3.0.3"); const documentedOperationCount = Object.values(docsJson.paths) .reduce((count, pathItem) => count + Object.keys(pathItem).length, 0); - expect(documentedOperationCount).toBe(9); + expect(documentedOperationCount).toBe(10); expect(Object.keys(docsJson.paths).every((pathName) => pathName.startsWith("/external/") || pathName.startsWith("/internal/model-gateway/") || pathName.startsWith("/usage/"))).toBe(true); @@ -51,6 +51,8 @@ it("assembles the management server and maps application and parser errors", asy .toEqual(jasmine.arrayContaining(["agentId", "senderId"])); expect(docsJson.paths["/internal/model-gateway/usage-events"].post.responses[200]).toBeDefined(); expect(docsJson.paths["/internal/model-gateway/usage-events"].post.responses[201]).toBeDefined(); + expect(docsJson.paths["/internal/model-gateway/quota-check"].post.requestBody.content["application/json"].schema.required) + .toEqual(["agentId", "senderId"]); expect(docsJson.paths["/usage/summary"].get.parameters) .toEqual(jasmine.arrayContaining([jasmine.objectContaining({ name: "group_by", in: "query" })])); expect(docsJson.paths["/external/agents"].get.parameters).toEqual( diff --git a/packages/daedalus-agent-management-web/server/test/model-gateway-usage-route.test.js b/packages/daedalus-agent-management-web/server/test/model-gateway-usage-route.test.js index 1219973..f3d5a22 100644 --- a/packages/daedalus-agent-management-web/server/test/model-gateway-usage-route.test.js +++ b/packages/daedalus-agent-management-web/server/test/model-gateway-usage-route.test.js @@ -69,6 +69,16 @@ function createApp({ remoteAddress = "" } = {}) { hasAccessToken: (token) => token === "internal-token" }, modelGatewayUsageService: usageService, + externalService: { + checkModelGatewayQuota: ({ agentId, senderId }) => ({ + allowed: agentId !== "blocked-agent" && senderId !== "blocked-user", + violations: agentId === "blocked-agent" + ? [{ subject: "agent", name: agentId, current: 10, limit: 10 }] + : senderId === "blocked-user" + ? [{ subject: "user", name: senderId, current: 20, limit: 20 }] + : [] + }) + }, auditLogService: { write() {} }, env: {} })); @@ -78,6 +88,31 @@ function createApp({ remoteAddress = "" } = {}) { return { app, db }; } +it("checks today's user and agent quotas for the OpenClaw run gate", async () => { + const { app, db } = createApp(); + const server = await listen(app); + try { + const baseUrl = `http://127.0.0.1:${server.address().port}`; + const allowed = await requestJson(`${baseUrl}/api/internal/model-gateway/quota-check`, { + method: "POST", + body: { agentId: "alpha", senderId: "alice" } + }); + expect(allowed.status).toBe(200); + expect(allowed.body).toEqual({ allowed: true, violations: [] }); + + const blocked = await requestJson(`${baseUrl}/api/internal/model-gateway/quota-check`, { + method: "POST", + body: { agentId: "blocked-agent", senderId: "alice" } + }); + expect(blocked.status).toBe(200); + expect(blocked.body.allowed).toBeFalse(); + expect(blocked.body.violations[0]).toEqual(jasmine.objectContaining({ subject: "agent", name: "blocked-agent" })); + } finally { + await close(server); + db.close(); + } +}); + it("writes idempotent gateway usage events and queries agent/sender dimensions", async () => { const { app, db } = createApp(); const server = await listen(app); diff --git a/packages/daedalus-agent-management-web/server/test/portal-service-alignment.test.js b/packages/daedalus-agent-management-web/server/test/portal-service-alignment.test.js index 5e9001b..d482876 100644 --- a/packages/daedalus-agent-management-web/server/test/portal-service-alignment.test.js +++ b/packages/daedalus-agent-management-web/server/test/portal-service-alignment.test.js @@ -319,6 +319,53 @@ it("aggregates dashboard agent and user usage from today's billable gateway even db.close(); }); +it("shows ledger token totals even when the legacy usage refresh is not ready", () => { + const db = new DatabaseSync(":memory:"); + db.exec(` + CREATE TABLE agent_sync_state (id INTEGER PRIMARY KEY, status TEXT, last_success_at TEXT); + CREATE TABLE usage_refresh_state (id INTEGER PRIMARY KEY, status TEXT, last_success_at TEXT); + CREATE TABLE agents ( + id INTEGER PRIMARY KEY, + slug TEXT NOT NULL, + agent_name TEXT NOT NULL, + status TEXT NOT NULL, + daily_limit INTEGER NOT NULL DEFAULT -1, + usage_snapshot_json TEXT NOT NULL DEFAULT '{}', + updated_at TEXT NOT NULL + ); + CREATE TABLE agent_users ( + id INTEGER PRIMARY KEY, + username TEXT NOT NULL, + daily_limit INTEGER NOT NULL DEFAULT -1 + ); + CREATE TABLE agent_templates (id INTEGER PRIMARY KEY); + CREATE TABLE skills (id INTEGER PRIMARY KEY); + CREATE TABLE business_systems (id INTEGER PRIMARY KEY); + CREATE TABLE system_invocation_logs (id INTEGER PRIMARY KEY, created_at TEXT); + CREATE TABLE model_usage_events ( + event_id TEXT PRIMARY KEY, + agent_id TEXT NOT NULL, + sender_id TEXT NOT NULL, + total_tokens INTEGER NOT NULL, + billable INTEGER NOT NULL DEFAULT 1, + created_at TEXT NOT NULL + ); + INSERT INTO agent_sync_state VALUES (1, 'success', '2026-01-01T00:00:00.000+08:00'); + INSERT INTO usage_refresh_state VALUES (1, 'running', NULL); + INSERT INTO agents (id, slug, agent_name, status, updated_at) + VALUES (1, 'finance-agent', 'Finance Agent', 'normal', '2026-01-01T00:00:00.000+08:00'); + `); + db.prepare(` + INSERT INTO model_usage_events (event_id, agent_id, sender_id, total_tokens, created_at) + VALUES ('ledger-total', 'finance-agent', 'alice', 42, ?) + `).run(toChinaISOString()); + + const dashboard = new PortalService({ db, objectStorage: {}, managementClient: {}, authService: {} }).getDashboard(); + + expect(dashboard.stats.today_tokens).toBe(42); + db.close(); +}); + it("returns component version info from the in-memory cache", () => { const service = new PortalService({ db: createVersionDb(), diff --git a/packages/daedalus-agent-management-web/server/test/user-quota.test.js b/packages/daedalus-agent-management-web/server/test/user-quota.test.js index 4609d50..aafb1a9 100644 --- a/packages/daedalus-agent-management-web/server/test/user-quota.test.js +++ b/packages/daedalus-agent-management-web/server/test/user-quota.test.js @@ -227,6 +227,24 @@ it("rejects session creation when either the user or agent reaches today's quota db.close(); }); +it("returns both user and digital employee violations for the model gateway run gate", () => { + const db = createQuotaDb(); + const service = createExternalService(db); + db.prepare("UPDATE agent_users SET daily_limit = 10 WHERE id = 1").run(); + db.prepare("UPDATE agents SET daily_limit = 20 WHERE id = 1").run(); + addUsage(db, { eventId: "gate-user-limit", totalTokens: 10 }); + addUsage(db, { eventId: "gate-agent-limit", senderId: "bob", totalTokens: 20 }); + + expect(service.checkModelGatewayQuota({ agentId: "finance-agent", senderId: "alice" })).toEqual({ + allowed: false, + violations: [ + jasmine.objectContaining({ subject: "agent", name: "finance-agent", current: 30, limit: 20 }), + jasmine.objectContaining({ subject: "user", name: "alice", current: 10, limit: 10 }) + ] + }); + db.close(); +}); + it("rechecks user quota for an existing session before resolving a business system", () => { const db = createQuotaDb(); const service = createExternalService(db); diff --git a/packages/daedalus-agent-management-web/src/pages/DashboardPage.jsx b/packages/daedalus-agent-management-web/src/pages/DashboardPage.jsx index cd3cbab..5cd6b50 100644 --- a/packages/daedalus-agent-management-web/src/pages/DashboardPage.jsx +++ b/packages/daedalus-agent-management-web/src/pages/DashboardPage.jsx @@ -114,7 +114,7 @@ export function DashboardPage({ refreshKey = 0 }) { ))} - +
- +
); - expect(await screen.findByText("用量表（用户）")).toBeInTheDocument(); - expect(screen.getByText("用量表（用户维度）")).toBeInTheDocument(); + expect(await screen.findByText("用量表（数字员工）")).toBeInTheDocument(); + expect(screen.getByText("用量表（用户）")).toBeInTheDocument(); expect(screen.getByTestId("dashboard-agent-usage-row-agent-1")).toBeInTheDocument(); expect(screen.queryByTestId("dashboard-agent-usage-row-agent-11")).not.toBeInTheDocument(); expect(screen.getByTestId("dashboard-user-usage-row-alice")).toBeInTheDocument(); diff --git a/test/case/upgrade/openclaw-upgrade-contract.test.js b/test/case/upgrade/openclaw-upgrade-contract.test.js index afb668d..f133ea9 100644 --- a/test/case/upgrade/openclaw-upgrade-contract.test.js +++ b/test/case/upgrade/openclaw-upgrade-contract.test.js @@ -161,6 +161,7 @@ test("preinstalled skills are self-contained for an offline runtime", async () = }); test("kernel startup performs versioned migration before plugin refresh", async () => { + const config = JSON.parse(await readRepoFile("docker/assets/config.json")); const startup = await readRepoFile("docker/assets/scripts/kernel-startup.sh"); const patchIndex = startup.indexOf('profile_step "patch_openclaw_config"'); const migrationIndex = startup.indexOf('profile_step "migrate_openclaw_state"'); @@ -172,6 +173,14 @@ test("kernel startup performs versioned migration before plugin refresh", async assert.match(startup, /doctor --non-interactive/); assert.match(startup, /blocked plugin candidate\|unknown channel id/); assert.match(startup, /\.plugins\.bundledDiscovery = \(.plugins.bundledDiscovery \/\/ "compat"\)/); + assert.equal( + config.plugins.entries["daedalus-agent-mqtt-channel"].hooks.allowConversationAccess, + true + ); + assert.match( + startup, + /\.plugins\.entries\["daedalus-agent-mqtt-channel"\]\.hooks\.allowConversationAccess = true/ + ); }); test("kernel startup synchronizes persisted MQTT account passwords from runtime credentials", async () => { -- Gitee From 813a725362a25ebab2222f39209ffa3883a71909 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=AE=81=E4=BC=9F?= Date: Wed, 2 Sep 2026 10:59:07 +0800 Subject: [PATCH 8/9] =?UTF-8?q?=E6=8C=89=E7=94=A8=E6=88=B7=E7=9A=84?= =?UTF-8?q?=E9=85=8D=E9=A2=9D=E9=99=90=E5=88=B6=E5=BC=80=E5=8F=91=E5=AE=8C?= =?UTF-8?q?=E6=AF=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- doc/system-feature-list.md | 13 +- docker/Dockerfile | 12 +- docker/README.md | 14 +- docker/assets/scripts/kernel-startup.sh | 69 ++++- docker/assets/scripts/startup.sh | 78 ++++- docker/docs/environment-variables.md | 7 +- docker/docs/spec.md | 4 +- docker/docs/startup-steps.md | 29 +- .../capabilities/agent/lifecycle-service.ts | 38 ++- .../capabilities/agent/service.ts | 7 +- .../capabilities/llm/service.ts | 136 +++++++- .../server/management-src/env.ts | 9 + .../tools/model-gateway-admin-client.ts | 54 ++++ .../server/management-src/types.ts | 3 + .../llm-model-config-service.test.ts | 46 +++ .../daedalus-agent-model-gateway/package.json | 10 + .../daedalus-agent-model-gateway/src/index.js | 27 ++ .../src/migrate.js | 54 ++++ .../src/server.js | 237 ++++++++++++++ .../daedalus-agent-model-gateway/src/store.js | 290 ++++++++++++++++++ .../test/migrate.test.js | 38 +++ .../test/server.test.js | 138 +++++++++ .../test/store.test.js | 37 +++ 23 files changed, 1299 insertions(+), 51 deletions(-) create mode 100644 packages/daedalus-agent-management-web/server/management-src/tools/model-gateway-admin-client.ts create mode 100644 packages/daedalus-agent-model-gateway/package.json create mode 100644 packages/daedalus-agent-model-gateway/src/index.js create mode 100644 packages/daedalus-agent-model-gateway/src/migrate.js create mode 100644 packages/daedalus-agent-model-gateway/src/server.js create mode 100644 packages/daedalus-agent-model-gateway/src/store.js create mode 100644 packages/daedalus-agent-model-gateway/test/migrate.test.js create mode 100644 packages/daedalus-agent-model-gateway/test/server.test.js create mode 100644 packages/daedalus-agent-model-gateway/test/store.test.js diff --git a/doc/system-feature-list.md b/doc/system-feature-list.md index 5d20688..709d3d3 100644 --- a/doc/system-feature-list.md +++ b/doc/system-feature-list.md @@ -6,12 +6,12 @@ ## 1. 系统定位与运行拓扑 -- [x][T] 提供一体化 Docker 部署，包含管理控制台、OpenClaw Gateway、MQTT Broker（Aedes）、MinIO、应用调用 socket service。 -- [x][T] 默认开放管理端口 `3030`、预留服务端口 `3031-3050`、MQTT `1883`、MinIO S3 API `9000`。 +- [x][T] 提供一体化 Docker 部署，包含管理控制台、Model Gateway、OpenClaw Gateway、MQTT Broker（Aedes）、MinIO、应用调用 socket service；Model Gateway 使用独立 `daedalus-model-gw` UID，Runtime API 仅绑定 `127.0.0.1:3040`，Admin API 仅通过 `/var/daedalus-agent/run/model-gateway-admin.sock` 提供。 +- [x][T] 默认开放管理端口 `3030`、预留服务端口 `3031-3039`、`3041-3050`（不暴露 Model Gateway `3040`）、MQTT `1883`、MinIO S3 API `9000`。 - [x] 使用命名 Docker volume 持久化 `/var/daedalus-agent`，保存 OpenClaw 状态、工作区、模板、Ontology、SQLite 数据、制品、日志、MQTT/MinIO 数据和运行状态。 - [x][T] 启动时校验 `env.json`、`config.json`；缺失时写入示例文件并要求用户检查后重启；拒绝用户覆盖由系统托管的内部路径、MQTT、MinIO、Topic 和 IPC 配置。 - [x][T] 自动生成并持久化 MQTT/MinIO 运行凭据；管理控制台可显示运行凭据，并支持管理员确认后重置两组密码和请求容器重启；容器启动时会将持久化 OpenClaw MQTT 账号密码同步为当前运行凭据。 -- [x] 以 `daedalus-agent` 运行 OpenClaw/Agent，以 `daedalus-svc` 运行守护进程、Broker、MinIO、管理端和 invoke service，通过共享组控制只读目录、日志和 socket 访问。 +- [x] 以 `daedalus-agent` 运行 OpenClaw/Agent，以 `daedalus-model-gw` 运行本机 Model Gateway，以 `daedalus-svc` 运行守护进程、Broker、MinIO、管理端和 invoke service，通过共享组控制只读目录、日志和 socket 访问。 - [x][T] 对 Agent 进程实施内网访问白名单和 iptables 限制；管理控制台可维护白名单，IP 地址变更在容器重启后生效。 - [x][T] 提供启动步骤耗时日志、子服务守护、服务重启、容器重启请求文件和快速重启保护。 - [x][T] 通过内置内容指纹执行 OpenClaw home、插件、Skill、模板和脚本的增量 seed/迁移；被替换内容先移入备份目录；Ontology 不参与 seed 覆盖。 @@ -42,13 +42,14 @@ ### 2.3 大语言模型管理 -- [x][T] Provider 新增、编辑、删除、列表；保存 OpenAI 兼容 `baseUrl` 和 API Key（仅保存掩码/指纹展示）。 +- [x][T] Provider 新增、编辑、删除、列表；管理端通过 Model Gateway Admin Unix Socket 保存 OpenAI 兼容 `baseUrl` 和 API Key，Provider Key 仅由网关 AES-256-GCM 加密保存，管理端和 OpenClaw 仅接收掩码/指纹或本机短期网关 Token。 - [x][T] Provider 连接测试，返回结构化成功/失败、HTTP/网络错误信息。 - [x][T] Model 新增、编辑、删除、按 Provider 筛选；维护模型 ID、名称、上下文长度、输出 Token 上限。 - [x][T] Model 连接测试，使用 OpenAI-compatible chat completion 验证配置和上下文。 - [x] 使用 `provider_id/model_id` 作为运行时模型引用；删除操作保护仍被 Agent/模板引用的记录。 - [x] 支持 Agent 主模型和备选模型；校验 Provider/Model 必须成对选择，备选模型不能与主模型相同。 - [x][T] Model Gateway 用量账本 `model_usage_events` 按 `agentId`、`senderId` 记录请求、输入/输出/缓存/总 Token、实际模型、计费和幂等字段；管理端提供按 Agent、senderId、Agent+senderId、模型、天和小时的统计查询，并以网关账本作为今日配额和总览用量事实来源，保留兼容 Agent 快照。 +- [x][T] Model Gateway Runtime API 提供 `GET /healthz`、`GET /readyz`、`GET /v1/models`、`POST /v1/chat/completions` 和 `POST /v1/embeddings`，校验 Bearer Runtime/Client Token、模型白名单和固定路由，支持非流式与 SSE 流式 OpenAI 兼容转发；上游错误、超时和用量事件统一归一化。 ### 2.4 员工模板管理 @@ -123,6 +124,7 @@ - [x][T] 持久外部会话 ID 使用 `s-` 加 30 位数字格式；同一用户-Agent 复用会话；会话创建校验 Agent 状态和用户授权。同步调用使用独立的一次性 MQTT 关联 ID，不属于持久外部会话。 - [x][T] 外部调用在每次解析业务系统时重新校验会话有效性、Agent 状态、业务系统状态和 Agent 业务系统策略。 - [x][T] 内部 CUI API：查询可用 Agent 目录；内部 invoke API：解析授权业务系统、获取业务系统 base URL、记录调用日志；Model Gateway 用量内部 API 写入幂等事件并提供每日配额检查；OpenClaw MQTT 插件通过 `llm_output` 落账并在 `before_agent_run` 阻断超限请求，提示明确指出超限用户或数字员工。 +- [x][T] OpenClaw 配置迁移后只保留 `daedalus-gateway` Provider、`http://127.0.0.1:3040/v1` 和 opaque Runtime Token；启动时将旧 OpenAI Provider 密钥迁移到网关加密存储并删除直连 Provider 配置；Agent 出口默认阻断外部 TCP `80/443` 直连模型 Provider。 - [x][T] 管理员用量 API：`GET /api/usage/summary` 支持按 Agent、senderId、Agent+senderId、模型和时间窗口聚合；`GET /api/usage/agents/{agentId}/senders` 查询指定 Agent 的 senderId 明细。 - [x][T] API 鉴权分层：管理端登录 Token、服务 Bearer Token、内部请求校验；localhost 请求按配置可走内部通道。 - [x][T] 外部 Cookie 缓存并按 `hzg` Provider 获取；业务系统调用结果写入统一调用日志和 Payload 制品。 @@ -168,7 +170,7 @@ | 范围 | 主要验证内容 | 代表测试位置 | | --- | --- | --- | -| 管理 Web 服务 | 路由、鉴权、用户/Agent/模板/Skill/LLM/MCP/业务系统 CRUD、内置模板编辑/重命名与删除保护、模板主/备选模型和 Agent 继承、Agent 用户访问策略（全部用户模式允许目录外未知用户名、指定用户模式仍需显式授权）、用户每日配额维护及今日用量/状态、用户改名与网关事件事务更新、Agent 多实例 MCP 选定服务同步及普通/沙箱 deny 投影、外部 Agent 目录匿名公开筛选与 `is_public` 标记、同步 Agent HTTP 接口（参数校验、服务鉴权、可选 Cookie、首次同步调用自动创建非管理员 Agent 用户、用户/Agent 任一配额超限时明确拒绝、临时会话业务系统调用、MQTT final-only 行为、临时 session 清理、成功后刷新用量并计入每日配额）、管理端 Agent 对话测试配额拦截、Model Gateway 用量事件幂等写入、按 Agent/senderId 聚合查询、Agent sender 明细、运行凭据密码重置及容器重启请求、OpenAPI/Swagger 文档外部接口和用量接口、审计、同步、健康检查、日志下载、SQLite 回滚和制品安全 | `packages/daedalus-agent-management-web/server/test/app-smoke.test.js`、`packages/daedalus-agent-management-web/server/test/model-gateway-usage-service.test.js`、`packages/daedalus-agent-management-web/server/test/model-gateway-usage-route.test.js`、`packages/daedalus-agent-management-web/server/test/catalog-sync-service.test.js`、`packages/daedalus-agent-management-web/server/test/db-reset-migration.test.js`、`packages/daedalus-agent-management-web/server/test/user-quota.test.js`、`packages/daedalus-agent-management-web/server/test/portal-service-alignment.test.js`、`packages/daedalus-agent-management-web/server/test/runtime-env-config-service.test.js`、`packages/daedalus-agent-management-web/server/test/runtime-settings-route.test.js`、`packages/daedalus-agent-management-web/server/test/topic-ping-service.test.js`、`packages/daedalus-agent-management-web/server/test/agent-user-policy.test.js`、`packages/daedalus-agent-management-web/server/test/external-service.test.js`、`packages/daedalus-agent-management-web/server/test/agent-service-alignment.test.js`、`packages/daedalus-agent-management-web/server/test/env-config.test.js`、`packages/daedalus-agent-management-web/server/test/mcp-service.test.js`、`packages/daedalus-agent-management-web/server/management-test/mcp-sync.test.ts`、其他 `server/test/`、`server/management-test/` | +| 管理 Web 服务 | 路由、鉴权、用户/Agent/模板/Skill/LLM/MCP/业务系统 CRUD、内置模板编辑/重命名与删除保护、模板主/备选模型和 Agent 继承、Agent 用户访问策略（全部用户模式允许目录外未知用户名、指定用户模式仍需显式授权）、用户每日配额维护及今日用量/状态、用户改名与网关事件事务更新、Agent 多实例 MCP 选定服务同步及普通/沙箱 deny 投影、外部 Agent 目录匿名公开筛选与 `is_public` 标记、同步 Agent HTTP 接口（参数校验、服务鉴权、可选 Cookie、首次同步调用自动创建非管理员 Agent 用户、用户/Agent 任一配额超限时明确拒绝、临时会话业务系统调用、MQTT final-only 行为、临时 session 清理、成功后刷新用量并计入每日配额）、管理端 Agent 对话测试配额拦截、Model Gateway Provider/Model Admin Socket 调用、Model Gateway 用量事件幂等写入、按 Agent/senderId 聚合查询、Agent sender 明细、运行凭据密码重置及容器重启请求、OpenAPI/Swagger 文档外部接口和用量接口、审计、同步、健康检查、日志下载、SQLite 回滚和制品安全 | `packages/daedalus-agent-management-web/server/test/app-smoke.test.js`、`packages/daedalus-agent-management-web/server/test/model-gateway-usage-service.test.js`、`packages/daedalus-agent-management-web/server/test/model-gateway-usage-route.test.js`、`packages/daedalus-agent-management-web/server/test/catalog-sync-service.test.js`、`packages/daedalus-agent-management-web/server/test/db-reset-migration.test.js`、`packages/daedalus-agent-management-web/server/test/user-quota.test.js`、`packages/daedalus-agent-management-web/server/test/portal-service-alignment.test.js`、`packages/daedalus-agent-management-web/server/test/runtime-env-config-service.test.js`、`packages/daedalus-agent-management-web/server/test/runtime-settings-route.test.js`、`packages/daedalus-agent-management-web/server/test/topic-ping-service.test.js`、`packages/daedalus-agent-management-web/server/test/agent-user-policy.test.js`、`packages/daedalus-agent-management-web/server/test/external-service.test.js`、`packages/daedalus-agent-management-web/server/test/agent-service-alignment.test.js`、`packages/daedalus-agent-management-web/server/test/env-config.test.js`、`packages/daedalus-agent-management-web/server/test/mcp-service.test.js`、`packages/daedalus-agent-management-web/server/management-test/mcp-sync.test.ts`、`packages/daedalus-agent-model-gateway/test/store.test.js`、`packages/daedalus-agent-model-gateway/test/server.test.js`、`packages/daedalus-agent-model-gateway/test/migrate.test.js`、其他 `server/test/`、`server/management-test/` | | 管理 Web 前端 | API 客户端鉴权/超时/错误归一化、同步门控、重启轮询、格式化、浏览器覆盖契约、员工与模板列表操作精简、内置模板编辑、名称/编辑详情入口、详情 tabs 保存流程、员工概览主模型/备选模型/每日配额快捷修改及打开状态下的 Token 用量同步刷新、用户每日限额/今日用量/状态列表、用户新增/编辑/删除、Dashboard 用户/Agent 用量表 10 条分页与零用量过滤、员工技能/业务系统/MCP 与模板设置布局一致及内置 Skill 隐藏、编辑页不展示同步调用 URL、模板文本文件切换后的编辑器可见性、设置页 API 文档入口、运行凭据重置按钮/确认框及“刷新状态”按钮 | `packages/daedalus-agent-management-web/src/**/*.test.js`、`packages/daedalus-agent-management-web/src/pages/AgentsPage.test.jsx`、`packages/daedalus-agent-management-web/src/pages/AgentUsersPage.test.jsx`、`packages/daedalus-agent-management-web/src/pages/DashboardPage.test.jsx`、`packages/daedalus-agent-management-web/test/browser-coverage.test.js`、`packages/daedalus-agent-management-web/test/run-browser-e2e.js`、`packages/daedalus-agent-management-web/test/integration/website-full.js` | | invoke CLI | 参数、JSON/body/query 来源、Binding schema、socket 协议、超时、大小限制、Provider 错误和调用日志 | `packages/daedalus-agent-apps-invoke-cli/test/` | | MQTT 插件 | 账号配置、旧配置兼容、环境变量覆盖、连接/断线重连、通配符、入站路由、普通消息串行队列、停止与 steer/tell 控制队列、active-run 非阻塞 steer、运行时接收 Promise 未完成时的确认与原 final 投递、原始会话目标、OpenClaw deliver 的 text/kind 无加工透传、出站 envelope、日志、Model Gateway `llm_output` 用量落账、`before_agent_run` 配额阻断和待写入同步 | `extensions/openclaw-plugins/daedalus-agent-mqtt-channel/src/*test.ts`、`extensions/openclaw-plugins/daedalus-agent-mqtt-channel/index.test.ts` | @@ -186,3 +188,4 @@ - [!] 内置模板可从名称或编辑按钮进入详情，支持重命名以及修改文件、Skill、业务系统、MCP 和模型配置，但不支持删除；所有模板的重命名均依赖远端模板创建/删除动作成功。 - [!] 业务调用应经过 invoke CLI/service；配套规则禁止 Agent 直接向业务系统发 HTTP 请求或手工启动 invoke service。 - [!] 集成/压测脚本默认使用 mock LLM；压测结果反映当前测试规格下的容量，不等同于真实模型 SLA。 +- [!] Model Gateway 当前是单容器本机 MVP：Runtime API 只监听 loopback，Admin API 依赖 Unix Socket 文件权限；尚未拆分独立容器/私有网络，也尚未实现可信 MQTT sender/session 签名上下文，因此用量 sender 默认记录为 `unknown`。 diff --git a/docker/Dockerfile b/docker/Dockerfile index 96bdf68..c2a8134 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -54,6 +54,9 @@ ENV DA_SERVICE_USER=daedalus-svc ENV DA_SERVICE_GROUP=daedalus-svc ENV DA_OPENCLAW_USER=daedalus-agent ENV DA_OPENCLAW_GROUP=daedalus-agent +ENV DA_MODEL_GATEWAY_USER=daedalus-model-gw +ENV DA_MODEL_GATEWAY_GROUP=daedalus-model-gw +ENV DA_MODEL_GATEWAY_HOME=/opt/daedalus-agent-model-gateway ENV DA_RUN_MODE=container ENV DA_DATA_DIR=/var/daedalus-agent/startup-state/data ENV DA_ONTOLOGY_DIR=/var/daedalus-agent/ontology @@ -244,8 +247,10 @@ RUN --mount=type=cache,target=/root/.cache/pip,sharing=locked \ RUN set -eux; \ groupadd --system "${DA_SHARED_GROUP}"; \ groupadd --system "${DA_SERVICE_GROUP}"; \ + groupadd --system "${DA_MODEL_GATEWAY_GROUP}"; \ useradd --system --gid "${DA_SERVICE_GROUP}" --groups "${DA_SHARED_GROUP}" --home-dir "${DA_ROOT}/management" --no-create-home --shell /bin/bash "${DA_SERVICE_USER}"; \ useradd --system --gid "${DA_OPENCLAW_GROUP}" --home-dir "${DA_OPENCLAW_HOME}" --no-create-home --shell /bin/bash "${DA_OPENCLAW_USER}"; \ + useradd --system --gid "${DA_MODEL_GATEWAY_GROUP}" --groups "${DA_SHARED_GROUP}" --home-dir "${DA_ROOT}/model-gateway" --no-create-home --shell /bin/bash "${DA_MODEL_GATEWAY_USER}"; \ mkdir -p "${DA_ROOT}/logs" "${DA_OPENCLAW_HOME}/logs"; \ chown "${DA_SERVICE_USER}:${DA_OPENCLAW_GROUP}" "${DA_ROOT}" "${DA_ROOT}/logs" "${DA_OPENCLAW_HOME}" "${DA_OPENCLAW_HOME}/logs"; \ chmod 2750 "${DA_ROOT}"; \ @@ -488,6 +493,9 @@ ENV DA_SERVICE_USER=daedalus-svc ENV DA_SERVICE_GROUP=daedalus-svc ENV DA_OPENCLAW_USER=daedalus-agent ENV DA_OPENCLAW_GROUP=daedalus-agent +ENV DA_MODEL_GATEWAY_USER=daedalus-model-gw +ENV DA_MODEL_GATEWAY_GROUP=daedalus-model-gw +ENV DA_MODEL_GATEWAY_HOME=/opt/daedalus-agent-model-gateway ENV DA_APPS_HOME=/opt/daedalus-agent-apps ENV DA_WEB_HOME=/opt/daedalus-agent-apps/web ENV DA_OPENCLAW_HOME=/var/daedalus-agent/.openclaw @@ -684,6 +692,8 @@ COPY --from=web-installer --chown=daedalus-svc:daedalus-agent /build/web/dist ${ COPY --from=web-installer --chown=daedalus-svc:daedalus-agent /build/web/server ${DA_WEB_HOME}/server COPY --from=web-installer --chown=daedalus-svc:daedalus-agent /build/web/scripts ${DA_WEB_HOME}/scripts +COPY --chown=daedalus-model-gw:daedalus-agent packages/daedalus-agent-model-gateway ${DA_MODEL_GATEWAY_HOME} + COPY --from=minio-image /usr/bin/minio /usr/local/bin/minio COPY --from=mc-image /usr/bin/mc /usr/local/bin/mc @@ -702,6 +712,6 @@ RUN set -eux; \ VOLUME ["/var/daedalus-agent"] -EXPOSE 3030 3031 3032 3033 3034 3035 3036 3037 3038 3039 3040 3041 3042 3043 3044 3045 3046 3047 3048 3049 3050 1883 9000 +EXPOSE 3030 3031 3032 3033 3034 3035 3036 3037 3038 3039 3041 3042 3043 3044 3045 3046 3047 3048 3049 3050 1883 9000 ENTRYPOINT ["/usr/bin/tini", "--", "/bin/bash", "/opt/daedalus-agent-assets/scripts/startup.sh"] diff --git a/docker/README.md b/docker/README.md index 4d5f5ba..22c1472 100644 --- a/docker/README.md +++ b/docker/README.md @@ -1,24 +1,24 @@ # DAEDALUS_AGENT Docker 镜像 -`docker/` 是当前唯一 Docker 入口。镜像把内置 MQTT、内置 MinIO、OpenClaw gateway、management web 和 `daedalus-agent-apps-invoke-cli serve` 放在同一个容器中运行，不再提供独立的 kernel/apps Docker 镜像。 +`docker/` 是当前唯一 Docker 入口。镜像把内置 MQTT、内置 MinIO、本机 Model Gateway、OpenClaw gateway、management web 和 `daedalus-agent-apps-invoke-cli serve` 放在同一个容器中运行，不再提供独立的 kernel/apps Docker 镜像。 -OpenClaw 进程以 `daedalus-agent` 身份运行；management web、invoke service、MQTT、MinIO 和顶层 supervisor 以 `daedalus-svc` 身份运行。`daedalus-agent` 的局域网访问受 Docker iptables 限制，只能访问本机、互联网和 `DA_INTERNAL_NETWORK_ALLOWLIST` 中的局域网地址；`daedalus-svc` 不受该限制，用于访问管理控制台和业务系统。 +OpenClaw 进程以 `daedalus-agent` 身份运行；Model Gateway 以 `daedalus-model-gw` 身份运行；management web、invoke service、MQTT、MinIO 和顶层 supervisor 以 `daedalus-svc` 身份运行。Model Gateway Runtime API 只绑定容器 loopback `127.0.0.1:3040`，Admin API 只通过 `/var/daedalus-agent/run/model-gateway-admin.sock` 提供，宿主机不应映射 `3040`。`daedalus-agent` 的局域网访问受 Docker iptables 限制，只能访问本机、互联网和 `DA_INTERNAL_NETWORK_ALLOWLIST` 中的局域网地址；`daedalus-svc` 不受该限制，用于访问管理控制台和业务系统。 ## 运行 ```bash docker volume create daedalus-agent-data -docker run -d --restart unless-stopped --name daedalus-agent --cap-add NET_ADMIN --network bridge --dns-search GRAPECITY.net -p 3030-3050:3030-3050 -p 1883:1883 -p 9000:9000 -v daedalus-agent-data:/var/daedalus-agent daedalus-agent-arm64:local +docker run -d --restart unless-stopped --name daedalus-agent --cap-add NET_ADMIN --network bridge --dns-search GRAPECITY.net -p 3030-3039:3030-3039 -p 3041-3050:3041-3050 -p 1883:1883 -p 9000:9000 -v daedalus-agent-data:/var/daedalus-agent daedalus-agent-arm64:local ``` Windows X64 示例： ```powershell docker volume create daedalus-agent-data -docker run -d --restart unless-stopped --name daedalus-agent --cap-add NET_ADMIN --network bridge --dns-search GRAPECITY.net -p 3030-3050:3030-3050 -p 1883:1883 -p 9000:9000 -v daedalus-agent-data:/var/daedalus-agent daedalus-agent-x64:local +docker run -d --restart unless-stopped --name daedalus-agent --cap-add NET_ADMIN --network bridge --dns-search GRAPECITY.net -p 3030-3039:3030-3039 -p 3041-3050:3041-3050 -p 1883:1883 -p 9000:9000 -v daedalus-agent-data:/var/daedalus-agent daedalus-agent-x64:local ``` -必须保留的宿主机开放端口：`1883`、`9000`、`3031-3050`。`3030` 默认用于 management web；`3031-3050` 预留给后续本地 HTTP 服务。 +必须保留的宿主机开放端口：`1883`、`9000`、`3031-3039`、`3041-3050`。`3030` 默认用于 management web；`3040` 是仅容器本机可访问的 Model Gateway Runtime API，不映射到宿主机。 运行数据首次需要提供： @@ -40,7 +40,7 @@ docker cp .\config.json daedalus-agent:/var/daedalus-agent/config.json docker start daedalus-agent ``` -容器会确保 `/var/daedalus-agent/run_after_startup.sh` 存在。该脚本在 MQTT、MinIO、OpenClaw gateway、management web 和 invoke service 全部就绪后执行，默认输出 `all is ok`；用户修改后重启容器即可生效。 +容器会确保 `/var/daedalus-agent/run_after_startup.sh` 存在。该脚本在 MQTT、MinIO、Model Gateway、OpenClaw gateway、management web 和 invoke service 全部就绪后执行，默认输出 `all is ok`；用户修改后重启容器即可生效。 最小 `env.json`： @@ -72,6 +72,8 @@ OpenClaw/agent 的局域网白名单通过管理控制台“系统设置 -> 基 /var/daedalus-agent/generated/runtime-env.json /var/daedalus-agent/generated/runtime-config.json /var/daedalus-agent/.openclaw/ +/var/daedalus-agent/model-gateway/data/ +/var/daedalus-agent/model-gateway/secrets/ /var/daedalus-agent/logs/ /var/daedalus-agent/management/data/ /var/daedalus-agent/ontology/ diff --git a/docker/assets/scripts/kernel-startup.sh b/docker/assets/scripts/kernel-startup.sh index 504a4c5..a2c2aee 100644 --- a/docker/assets/scripts/kernel-startup.sh +++ b/docker/assets/scripts/kernel-startup.sh @@ -58,6 +58,7 @@ OPENCLAW_RESTART_DELAY_SECONDS="${OPENCLAW_RESTART_DELAY_SECONDS:-2}" OPENCLAW_FATAL_EXIT_CODE="${OPENCLAW_FATAL_EXIT_CODE:-78}" OPENCLAW_MAX_RAPID_RESTARTS="${OPENCLAW_MAX_RAPID_RESTARTS:-5}" OPENCLAW_RAPID_RESTART_WINDOW_SECONDS="${OPENCLAW_RAPID_RESTART_WINDOW_SECONDS:-60}" +DA_AGENT_NETWORK_GATEWAY_ONLY="${DA_AGENT_NETWORK_GATEWAY_ONLY:-true}" declare -ar RETIRED_BUILTIN_SKILLS=(video-frames ffmpeg-video-editor mcporter) OPENCLAW_MIGRATION_STAMP="${DA_OPENCLAW_HOME}/state/daedalus-agent-openclaw-migrated.version" RUNTIME_ENV_KEYS=() @@ -808,6 +809,13 @@ add_agent_netfilter_rules() { fi done <<< "$allowed_endpoints" + if [[ "${DA_AGENT_NETWORK_GATEWAY_ONLY,,}" == "true" || "${DA_AGENT_NETWORK_GATEWAY_ONLY}" == "1" || "${DA_AGENT_NETWORK_GATEWAY_ONLY,,}" == "yes" ]]; then + for restricted_port in 80 443; do + "$command_name" -w -A "$chain_name" -p tcp --dport "$restricted_port" -j REJECT \ + || fail_agent_internal_network_blocker "${command_name} cannot block direct model provider traffic on port ${restricted_port}" + done + fi + for cidr in "${cidrs[@]}"; do "$command_name" -w -A "$chain_name" -d "$cidr" -j REJECT \ || fail_agent_internal_network_blocker "${command_name} cannot add internal network block for ${cidr}" @@ -2216,6 +2224,8 @@ patch_openclaw_config() { local config_source="$RUNTIME_CONFIG_JSON" local tmp_json local gateway_auth_token="${DA_OPENCLAW_GATEWAY_AUTH_TOKEN:-daedalus-agent-local-gateway-token}" + local model_gateway_url="${DA_MODEL_GATEWAY_URL:-http://127.0.0.1:3040/v1}" + local model_gateway_runtime_token="${DA_MODEL_GATEWAY_RUNTIME_TOKEN:-}" local gateway_max_concurrent local mqtt_password="${DA_MQTT_CHANNEL_PASSWORD:-}" @@ -2238,21 +2248,58 @@ patch_openclaw_config() { --arg workspace "$DA_OPENCLAW_WORKSPACE_DIR" \ --arg qmdCommand "$QMD_COMMAND" \ --arg gatewayAuthToken "$gateway_auth_token" \ + --arg modelGatewayUrl "$model_gateway_url" \ + --arg modelGatewayRuntimeToken "$model_gateway_runtime_token" \ --arg mqttPassword "$mqtt_password" \ --arg seedMqttPluginPath "$SEED_MQTT_PLUGIN_PATH" \ --argjson gatewayMaxConcurrent "$gateway_max_concurrent" \ ' - if (.models.providers? | type) == "object" then - .models.providers |= with_entries( - if (.value.api? == "openai-completions") and ((.value.baseUrl? | type) == "string") then - .value.baseUrl |= sub("/chat/completions/?$"; "") | .value.baseUrl |= sub("/completions/?$"; "") - else - . - end - ) - else - . - end + def gatewayModelRef: + if (type == "string") then + if startswith("daedalus-gateway/") then . else ("daedalus-gateway/" + .) end + else . end; + (.models.providers // {}) as $legacyProviders + | ([ + $legacyProviders + | to_entries[] + | .key as $providerId + | (.value.models // [])[] + | select((.id // "") != "") + | { + id: (if $providerId == "daedalus-gateway" then .id else ($providerId + "/" + .id) end), + name: (.name // .id), + contextTokens: .contextTokens, + maxTokens: .maxTokens + } + | with_entries(select(.value != null)) + ] | unique_by(.id)) as $gatewayModels + | .models = ((.models // {}) + { + mode: "merge", + providers: { + "daedalus-gateway": { + api: "openai-completions", + baseUrl: $modelGatewayUrl, + apiKey: $modelGatewayRuntimeToken, + models: $gatewayModels + } + } + }) + | if ((.agents.list? | type) == "array") then + .agents.list |= map( + if ((.model? | type) == "string") then + .model |= gatewayModelRef + elif ((.model? | type) == "object") then + .model.primary |= gatewayModelRef + | if ((.model.fallbacks? | type) == "array") then .model.fallbacks |= map(gatewayModelRef) else . end + else . end + ) + else . end + | if ((.agents.defaults.model? | type) == "string") then + .agents.defaults.model |= gatewayModelRef + elif ((.agents.defaults.model? | type) == "object") then + .agents.defaults.model.primary |= gatewayModelRef + | if ((.agents.defaults.model.fallbacks? | type) == "array") then .agents.defaults.model.fallbacks |= map(gatewayModelRef) else . end + else . end | .agents.defaults.workspace = $workspace | .agents.defaults.maxConcurrent = $gatewayMaxConcurrent diff --git a/docker/assets/scripts/startup.sh b/docker/assets/scripts/startup.sh index 0ace5d6..ec76ab1 100644 --- a/docker/assets/scripts/startup.sh +++ b/docker/assets/scripts/startup.sh @@ -19,6 +19,14 @@ DA_SERVICE_USER="${DA_SERVICE_USER:-daedalus-svc}" DA_SERVICE_GROUP="${DA_SERVICE_GROUP:-daedalus-svc}" DA_OPENCLAW_USER="${DA_OPENCLAW_USER:-daedalus-agent}" DA_OPENCLAW_GROUP="${DA_OPENCLAW_GROUP:-daedalus-agent}" +DA_MODEL_GATEWAY_USER="${DA_MODEL_GATEWAY_USER:-daedalus-model-gw}" +DA_MODEL_GATEWAY_GROUP="${DA_MODEL_GATEWAY_GROUP:-daedalus-model-gw}" +DA_MODEL_GATEWAY_PORT="${DA_MODEL_GATEWAY_PORT:-3040}" +DA_MODEL_GATEWAY_HOST="127.0.0.1" +DA_MODEL_GATEWAY_DATA_DIR="${DA_MODEL_GATEWAY_DATA_DIR:-${DA_ROOT}/model-gateway/data}" +DA_MODEL_GATEWAY_SECRET_DIR="${DA_MODEL_GATEWAY_SECRET_DIR:-${DA_ROOT}/model-gateway/secrets}" +DA_MODEL_GATEWAY_ADMIN_SOCKET="${DA_MODEL_GATEWAY_ADMIN_SOCKET:-${DA_ROOT}/run/model-gateway-admin.sock}" +DA_MODEL_GATEWAY_ENTRY="${DA_MODEL_GATEWAY_ENTRY:-/opt/daedalus-agent-model-gateway/src/index.js}" GENERATED_DIR="${DA_ROOT}/generated" RUNTIME_ENV_JSON="${GENERATED_DIR}/runtime-env.json" RUNTIME_CONFIG_JSON="${GENERATED_DIR}/runtime-config.json" @@ -46,11 +54,12 @@ MINIO_PID="" MQTT_PID="" KERNEL_PID="" APPS_PID="" +MODEL_GATEWAY_PID="" CONTAINER_RESTART_WATCHER_PID="" KERNEL_LOG_READY_OFFSET=0 OPENCLAW_LOG_READY_OFFSET=0 SHUTTING_DOWN=false -STARTUP_STEP_TOTAL=14 +STARTUP_STEP_TOTAL=15 STARTUP_STEP_INDEX=0 STARTUP_ACTIVE_STEP_INDEX=0 STARTUP_ACTIVE_STEP_NAME="" @@ -187,13 +196,13 @@ trap startup_exit_trap EXIT stop_children() { local pid - for pid in "${CONTAINER_RESTART_WATCHER_PID:-}" "${APPS_PID:-}" "${KERNEL_PID:-}" "${MINIO_PID:-}" "${MQTT_PID:-}"; do + for pid in "${CONTAINER_RESTART_WATCHER_PID:-}" "${APPS_PID:-}" "${KERNEL_PID:-}" "${MODEL_GATEWAY_PID:-}" "${MINIO_PID:-}" "${MQTT_PID:-}"; do if [[ -n "${pid}" ]] && kill -0 "${pid}" 2>/dev/null; then kill "${pid}" 2>/dev/null || true fi done - for pid in "${CONTAINER_RESTART_WATCHER_PID:-}" "${APPS_PID:-}" "${KERNEL_PID:-}" "${MINIO_PID:-}" "${MQTT_PID:-}"; do + for pid in "${CONTAINER_RESTART_WATCHER_PID:-}" "${APPS_PID:-}" "${KERNEL_PID:-}" "${MODEL_GATEWAY_PID:-}" "${MINIO_PID:-}" "${MQTT_PID:-}"; do if [[ -n "${pid}" ]]; then wait "${pid}" 2>/dev/null || true fi @@ -393,6 +402,9 @@ ensure_directories() { ensure_service_directory "${DA_ROOT}/mqtt/aedes/config" ensure_service_directory "${DA_ROOT}/mqtt/aedes/persistence" mkdir -p "${MINIO_DATA_DIR}" + mkdir -p "${DA_MODEL_GATEWAY_DATA_DIR}" "${DA_MODEL_GATEWAY_SECRET_DIR}" + chown "${DA_MODEL_GATEWAY_USER}:${DA_MODEL_GATEWAY_GROUP}" "${DA_MODEL_GATEWAY_DATA_DIR}" "${DA_MODEL_GATEWAY_SECRET_DIR}" 2>/dev/null || true + chmod 0700 "${DA_MODEL_GATEWAY_DATA_DIR}" "${DA_MODEL_GATEWAY_SECRET_DIR}" 2>/dev/null || true } ensure_owned_directory() { @@ -672,6 +684,7 @@ generate_runtime_files() { node - "${USER_ENV_JSON}" "${USER_CONFIG_JSON}" "${CREDENTIALS_JSON}" "${GENERATED_DIR}" "${DA_ROOT}" "${DA_WEB_DATA_DIR}/management-console.db" <<'NODE' const fs = require("node:fs"); const path = require("node:path"); +const crypto = require("node:crypto"); const [userEnvPath, userConfigPath, credentialsPath, generatedDir, productRoot, databasePath] = process.argv.slice(2); function readJson(filePath) { @@ -770,6 +783,16 @@ const shared = { DA_ONTOLOGY_DIR: path.posix.join(productRoot, "ontology"), DA_TMP_DIR: path.posix.join(productRoot, ".openclaw/tmp"), DA_WEB_DATA_DIR: path.posix.join(productRoot, "management/data"), + DA_MODEL_GATEWAY_HOST: "127.0.0.1", + DA_MODEL_GATEWAY_PORT: "3040", + DA_MODEL_GATEWAY_URL: "http://127.0.0.1:3040/v1", + DA_MODEL_GATEWAY_DATA_DIR: path.posix.join(productRoot, "model-gateway/data"), + DA_MODEL_GATEWAY_SECRET_DIR: path.posix.join(productRoot, "model-gateway/secrets"), + DA_MODEL_GATEWAY_MASTER_KEY_FILE: path.posix.join(productRoot, "model-gateway/secrets/master.key"), + DA_MODEL_GATEWAY_ADMIN_SOCKET: path.posix.join(productRoot, "run/model-gateway-admin.sock"), + DA_MODEL_GATEWAY_RUNTIME_TOKEN: String(value("DA_MODEL_GATEWAY_RUNTIME_TOKEN", crypto.randomBytes(32).toString("base64url"))), + DA_MODEL_GATEWAY_USAGE_ENDPOINT: "http://127.0.0.1:3030/api/internal/model-gateway/usage-events", + DA_AGENT_NETWORK_GATEWAY_ONLY: String(valueWithDefault("DA_AGENT_NETWORK_GATEWAY_ONLY", "true")), DA_INTERNAL_NETWORK_ALLOWLIST: process.env.DA_INTERNAL_NETWORK_ALLOWLIST ?? readRuntimeSetting("internal_network_allowlist") ?? valueWithDefault("DA_INTERNAL_NETWORK_ALLOWLIST", ""), @@ -876,6 +899,43 @@ start_minio() { wait_for_http "http://127.0.0.1:9000/minio/health/ready" 90 } +start_model_gateway() { + local runtime_token management_port + runtime_token="$(jq -r '.DA_MODEL_GATEWAY_RUNTIME_TOKEN' "${RUNTIME_ENV_JSON}")" + management_port="$(jq -r '.DA_MANAGEMENT_CONSOLE_PORT // 3030' "${RUNTIME_ENV_JSON}")" + rm -f "${DA_MODEL_GATEWAY_ADMIN_SOCKET}" + log "starting local model gateway" + ( + export DA_MODEL_GATEWAY_HOST="${DA_MODEL_GATEWAY_HOST}" + export DA_MODEL_GATEWAY_PORT="${DA_MODEL_GATEWAY_PORT}" + export DA_MODEL_GATEWAY_DATA_DIR="${DA_MODEL_GATEWAY_DATA_DIR}" + export DA_MODEL_GATEWAY_SECRET_DIR="${DA_MODEL_GATEWAY_SECRET_DIR}" + export DA_MODEL_GATEWAY_MASTER_KEY_FILE="${DA_MODEL_GATEWAY_SECRET_DIR}/master.key" + export DA_MODEL_GATEWAY_ADMIN_SOCKET="${DA_MODEL_GATEWAY_ADMIN_SOCKET}" + export DA_MODEL_GATEWAY_RUNTIME_TOKEN="${runtime_token}" + export DA_MODEL_GATEWAY_USAGE_ENDPOINT="http://127.0.0.1:${management_port}/api/internal/model-gateway/usage-events" + for migration_config in "${RUNTIME_CONFIG_JSON}" "${DA_OPENCLAW_HOME}/openclaw.json"; do + if [[ -f "${migration_config}" ]]; then + runuser --preserve-environment -u "${DA_MODEL_GATEWAY_USER}" -- node "${DA_MODEL_GATEWAY_ENTRY%/src/index.js}/src/migrate.js" "${migration_config}" || true + fi + done + exec runuser --preserve-environment -u "${DA_MODEL_GATEWAY_USER}" -- node "${DA_MODEL_GATEWAY_ENTRY}" + ) > >(write_daily_package_log "model-gateway" "[model-gateway]") 2>&1 & + MODEL_GATEWAY_PID=$! + printf '%s\n' "${MODEL_GATEWAY_PID}" > "${RUN_DIR}/model-gateway.pid" + wait_for_http "http://${DA_MODEL_GATEWAY_HOST}:${DA_MODEL_GATEWAY_PORT}/readyz" 60 + # 管理 socket 只授予 management service，避免 Agent 用户直接调用网关管理面。 + chown "${DA_SERVICE_USER}:${DA_SERVICE_GROUP}" "${DA_MODEL_GATEWAY_ADMIN_SOCKET}" 2>/dev/null || true + chmod 0660 "${DA_MODEL_GATEWAY_ADMIN_SOCKET}" 2>/dev/null || true +} + +wait_for_model_gateway() { + local token + token="$(jq -r '.DA_MODEL_GATEWAY_RUNTIME_TOKEN' "${RUNTIME_ENV_JSON}")" + wait_for_http "http://${DA_MODEL_GATEWAY_HOST}:${DA_MODEL_GATEWAY_PORT}/readyz" 60 + curl -fsS -H "Authorization: Bearer ${token}" "http://${DA_MODEL_GATEWAY_HOST}:${DA_MODEL_GATEWAY_PORT}/v1/models" >/dev/null +} + init_minio_buckets() { local access_key secret_key bucket access_key="$(jq -r '.DA_S3_ACCESS_KEY_ID' "${RUNTIME_ENV_JSON}")" @@ -1218,6 +1278,15 @@ restart_dead_children() { restarted=true fi + if [[ -n "${MODEL_GATEWAY_PID}" ]] && ! kill -0 "${MODEL_GATEWAY_PID}" 2>/dev/null; then + log "model gateway exited with code ${exit_code}; restarting" + MODEL_GATEWAY_PID="" + rm -f "${RUN_DIR}/model-gateway.pid" + sleep "$(restart_delay_seconds)" + start_model_gateway + restarted=true + fi + if [[ -n "${KERNEL_PID}" ]] && ! kill -0 "${KERNEL_PID}" 2>/dev/null; then restart_kernel "${exit_code}" restarted=true @@ -1238,7 +1307,7 @@ supervise_children() { while [[ "${SHUTTING_DOWN}" != "true" ]]; do set +e - wait -n "${KERNEL_PID}" "${APPS_PID}" "${MINIO_PID}" "${MQTT_PID}" + wait -n "${KERNEL_PID}" "${APPS_PID}" "${MODEL_GATEWAY_PID}" "${MINIO_PID}" "${MQTT_PID}" exit_code=$? set -e @@ -1265,6 +1334,7 @@ main() { profile_step "start_mqtt" start_mqtt profile_step "start_minio" start_minio profile_step "init_minio_buckets" init_minio_buckets + profile_step "start_model_gateway" start_model_gateway profile_step "start_kernel" start_kernel profile_step "start_apps" start_apps profile_step "wait_for_kernel" wait_for_kernel diff --git a/docker/docs/environment-variables.md b/docker/docs/environment-variables.md index a69c621..d74a548 100644 --- a/docker/docs/environment-variables.md +++ b/docker/docs/environment-variables.md @@ -6,7 +6,7 @@ DAEDALUS_AGENT Docker 支持通过 `/var/daedalus-agent/env.json` 配置少量 | 变量 | 默认值 | 说明 | | --- | --- | --- | -| `DA_MANAGEMENT_CONSOLE_PORT` | `3030` | management web 监听端口。建议宿主机映射整个 `3030-3050` 区间。 | +| `DA_MANAGEMENT_CONSOLE_PORT` | `3030` | management web 监听端口。建议宿主机映射 `3030-3039`、`3041-3050`，不要映射本机 Model Gateway 的 `3040`。 | | `DA_MANAGEMENT_CONSOLE_PUBLIC_BASE_URL` | 空（回退 `http://127.0.0.1:`） | 对外同步 Agent HTTP 接口生成 URL 使用的管理控制台基址；应为 `http` 或 `https` URL，末尾斜杠可省略，非法值按回退地址处理。 | | `DA_OPENCLAW_GATEWAY_MAX_CONCURRENT` | `128` | OpenClaw gateway 执行 lane 并发数，启动时写入 `openclaw.json` 的 `agents.defaults.maxConcurrent`。 | | `DA_MANAGEMENT_WEBSITE_TIMEOUT` | `180` | management web 访问内部管理动作的超时时间，单位秒。 | @@ -14,6 +14,8 @@ DAEDALUS_AGENT Docker 支持通过 `/var/daedalus-agent/env.json` 配置少量 | `DA_SYSTEM_STATUS_REFRESH_INTERVAL_MS` | `60000` | 首次采集完成后系统状态缓存的刷新间隔，单位毫秒。 | | `DA_SYSTEM_STATUS_TIMEOUT_MS` | `5000` | 单次系统状态采集调用的超时时间，单位毫秒。 | +Model Gateway 固定使用 `127.0.0.1:3040`，由启动脚本生成 `DA_MODEL_GATEWAY_RUNTIME_TOKEN` 并写入 OpenClaw 的 `daedalus-gateway` Provider。Provider 上游地址和 API Key 应通过管理控制台保存，Key 只进入网关加密存储，不写入 `openclaw.json`。 + 最小 `env.json`： ```json @@ -51,6 +53,9 @@ DAEDALUS_AGENT Docker 支持通过 `/var/daedalus-agent/env.json` 配置少量 | 变量 | 固定值 | 说明 | | --- | --- | --- | | `DA_ROOT` | `/var/daedalus-agent` | DAEDALUS_AGENT 容器运行根目录。 | +| `DA_MODEL_GATEWAY_HOST` | `127.0.0.1` | Model Gateway Runtime API 绑定地址，固定为本机 loopback。 | +| `DA_MODEL_GATEWAY_PORT` | `3040` | Model Gateway Runtime API 端口，不应映射到宿主机。 | +| `DA_MODEL_GATEWAY_ADMIN_SOCKET` | `/var/daedalus-agent/run/model-gateway-admin.sock` | Model Gateway Admin Unix Socket；启动后仅 `daedalus-svc` 可读写。 | | `DA_DATA_DIR` | `/var/daedalus-agent/startup-state/data` | 启动状态和 seed 指纹目录。 | | `DA_OPENCLAW_HOME` | `/var/daedalus-agent/.openclaw` | OpenClaw 持久化目录。 | | `DA_OPENCLAW_WORKSPACE_DIR` | `/var/daedalus-agent/.openclaw/workspaces/default` | 默认 workspace。 | diff --git a/docker/docs/spec.md b/docker/docs/spec.md index 7841ca5..42040cd 100644 --- a/docker/docs/spec.md +++ b/docker/docs/spec.md @@ -8,15 +8,17 @@ - 内置 MQTT broker，监听 `0.0.0.0:1883`。 - 内置 MinIO S3 API，监听 `:9000`。 +- 本机 Model Gateway，Runtime API 监听 `127.0.0.1:3040`，Admin API 监听 Unix Socket `/var/daedalus-agent/run/model-gateway-admin.sock`。 - OpenClaw gateway supervisor。 - management web。 - `daedalus-agent-apps-invoke-cli serve`。 -镜像声明开放端口：`3030-3050`、`1883`、`9000`。宿主机映射必须保留 `1883`、`9000`、`3031-3050`；`3030` 默认用于 management web。 +镜像声明开放端口：`3030-3039`、`3041-3050`、`1883`、`9000`。`3040` 不声明也不映射，Model Gateway Runtime API 只供容器内 loopback 调用。宿主机映射必须保留 `1883`、`9000`、`3031-3039`、`3041-3050`；`3030` 默认用于 management web。 ## 双用户机制 - `daedalus-agent`：运行 OpenClaw gateway 和 agent 相关命令。局域网访问受 iptables 限制，只能访问本机、互联网和 `DA_INTERNAL_NETWORK_ALLOWLIST` 白名单中的局域网地址。 +- `daedalus-model-gw`：运行本机 Model Gateway，读取加密 Provider Key 并负责固定上游路由；不对外监听。 - `daedalus-svc`：运行 supervisor、MQTT、MinIO、management web、invoke service、启动 hook。可访问全部网络接口。 - 共享组统一为 `daedalus-agent`，用于只读共享目录和日志/socket 访问控制。 diff --git a/docker/docs/startup-steps.md b/docker/docs/startup-steps.md index 842b6f6..ebca939 100644 --- a/docker/docs/startup-steps.md +++ b/docker/docs/startup-steps.md @@ -6,20 +6,21 @@ | 序号 | stepName | 说明 | | --- | --- | --- | -| 1/14 | `require_input_files` | 检查 `/var/daedalus-agent/env.json` 和 `config.json`；缺失时播种 sample 文件并退出，存在时校验为 JSON object，并拒绝用户覆盖托管变量。 | -| 2/14 | `ensure_directories` | 创建统一运行目录：`.openclaw`、`logs`、`generated`、`management/data`、`ontology`、`workspace-templates`、`minio`、`mqtt/aedes`、`startup-state`、`scripts`、`run`，并规范用户/组/权限。 | -| 3/14 | `ensure_credentials` | 生成或复用 MQTT/MinIO 随机密码，写入 `credentials.json`。 | -| 4/14 | `generate_runtime_files` | 合并容器环境、SQLite 运行设置、用户环境、凭据和配置，生成 `generated/runtime-env.json` 与 `generated/runtime-config.json`。 | -| 5/14 | `sync_optional_user_files` | 同步可选 `agent_workspace` 到 `workspace-templates/default`。 | -| 6/14 | `ensure_default_startup_hook` | 确保 `/var/daedalus-agent/run_after_startup.sh` 存在；缺失时写入默认脚本。 | -| 7/14 | `start_mqtt` | 启动内置 MQTT broker 并等待 1883 可用。 | -| 8/14 | `start_minio` | 启动内置 MinIO 并等待健康检查通过。 | -| 9/14 | `init_minio_buckets` | 创建管理端和 agent 的 inbox/outbox bucket。 | -| 10/14 | `start_kernel` | 注入统一运行环境并启动 OpenClaw supervisor。 | -| 11/14 | `start_apps` | 以 `daedalus-svc` 启动 management web 和 `daedalus-agent-apps-invoke-cli serve`；invoke service 监听 `/var/daedalus-agent/run/app-invoke.sock`。 | -| 12/14 | `wait_for_kernel` | 等待 OpenClaw gateway ready 日志出现。 | -| 13/14 | `wait_for_apps` | 等待 management web HTTP 可访问，并确认 invoke service pid 存活、`/var/daedalus-agent/run/app-invoke.sock` 权限可供 agent 连接。 | -| 14/14 | `run_after_startup` | 在所有组件就绪后执行 `/var/daedalus-agent/run_after_startup.sh`。 | +| 1/15 | `require_input_files` | 检查 `/var/daedalus-agent/env.json` 和 `config.json`；缺失时播种 sample 文件并退出，存在时校验为 JSON object，并拒绝用户覆盖托管变量。 | +| 2/15 | `ensure_directories` | 创建统一运行目录：`.openclaw`、`logs`、`generated`、`management/data`、`model-gateway/data`、`model-gateway/secrets`、`ontology`、`workspace-templates`、`minio`、`mqtt/aedes`、`startup-state`、`scripts`、`run`，并规范用户/组/权限。 | +| 3/15 | `ensure_credentials` | 生成或复用 MQTT/MinIO 随机密码，写入 `credentials.json`。 | +| 4/15 | `generate_runtime_files` | 合并容器环境、SQLite 运行设置、用户环境、凭据和配置，生成 `generated/runtime-env.json` 与 `generated/runtime-config.json`。 | +| 5/15 | `sync_optional_user_files` | 同步可选 `agent_workspace` 到 `workspace-templates/default`。 | +| 6/15 | `ensure_default_startup_hook` | 确保 `/var/daedalus-agent/run_after_startup.sh` 存在；缺失时写入默认脚本。 | +| 7/15 | `start_mqtt` | 启动内置 MQTT broker 并等待 1883 可用。 | +| 8/15 | `start_minio` | 启动内置 MinIO 并等待健康检查通过。 | +| 9/15 | `init_minio_buckets` | 创建管理端和 agent 的 inbox/outbox bucket。 | +| 10/15 | `start_model_gateway` | 以 `daedalus-model-gw` 启动本机 Model Gateway，迁移旧 OpenClaw Provider 配置，等待 `127.0.0.1:3040/readyz`，并将 Admin socket 权限收紧到 `daedalus-svc`。 | +| 11/15 | `start_kernel` | 注入统一运行环境并启动 OpenClaw supervisor；配置中的模型统一指向 `daedalus-gateway`。 | +| 12/15 | `start_apps` | 以 `daedalus-svc` 启动 management web 和 `daedalus-agent-apps-invoke-cli serve`；invoke service 监听 `/var/daedalus-agent/run/app-invoke.sock`。 | +| 13/15 | `wait_for_kernel` | 等待 OpenClaw gateway ready 日志出现。 | +| 14/15 | `wait_for_apps` | 等待 management web HTTP 可访问，并确认 invoke service pid 存活、`/var/daedalus-agent/run/app-invoke.sock` 权限可供 agent 连接。 | +| 15/15 | `run_after_startup` | 在所有组件就绪后执行 `/var/daedalus-agent/run_after_startup.sh`。 | 步骤完成后，顶层脚本进入服务守护循环。后续 MQTT、MinIO、OpenClaw supervisor 或 apps supervisor 异常退出时，会按 `DA_SERVICE_RESTART_DELAY_SECONDS` 延迟后重启。 diff --git a/packages/daedalus-agent-management-web/server/management-src/capabilities/agent/lifecycle-service.ts b/packages/daedalus-agent-management-web/server/management-src/capabilities/agent/lifecycle-service.ts index 63d6094..b7ec889 100644 --- a/packages/daedalus-agent-management-web/server/management-src/capabilities/agent/lifecycle-service.ts +++ b/packages/daedalus-agent-management-web/server/management-src/capabilities/agent/lifecycle-service.ts @@ -72,6 +72,12 @@ function firstText(...values: unknown[]): string { return ""; } +function addGatewayPrefix(value: string): string { + const normalized = String(value || "").trim(); + if (!normalized || normalized.startsWith("daedalus-gateway/")) return normalized; + return `daedalus-gateway/${normalized}`; +} + function addUsageByAgent(target: Map, agentId: string, source: unknown): void { const normalizedAgentId = agentId.trim(); if (!normalizedAgentId) { @@ -148,7 +154,7 @@ export class AgentLifecycleService { name: agent?.name ?? agentId, workspace: managed?.workspace ?? agent?.workspace ?? "", templateName: managed?.templateName, - model: agent?.model ?? null, + model: this.toManagementModel(agent?.model ?? null), status: enabled ? "active" : "disabled", health: enabled ? "normal" : "disabled", usage: { @@ -262,7 +268,7 @@ export class AgentLifecycleService { } async setAgentModel(params: AgentModelSetParams): Promise { - const model = normalizeModelConfig(params.model); + const model = this.toRuntimeModel(normalizeModelConfig(params.model)); const config = await this.readOpenClawConfig(); const agents = Array.isArray(config.agents?.list) ? config.agents.list : []; const agent = agents.find((entry: { id?: string }) => entry?.id === params.agentId); @@ -275,7 +281,7 @@ export class AgentLifecycleService { return { agentId: params.agentId, - model + model: this.toManagementModel(model) }; } @@ -385,7 +391,7 @@ export class AgentLifecycleService { private async createAgentByConfigMutation(params: ResolvedAgentCreateOptions): Promise { const config = await this.readOpenClawConfig(); const agentDir = path.join(this.env.openclawHome, "agents", params.agentId, "agent"); - const model = params.model ? normalizeModelConfig(params.model) : undefined; + const model = params.model ? this.toRuntimeModel(normalizeModelConfig(params.model)) : undefined; config.agents ??= {}; const existingAgents = Array.isArray(config.agents.list) ? config.agents.list : []; @@ -628,7 +634,7 @@ export class AgentLifecycleService { agent.name = name; } if (params.model !== undefined) { - agent.model = normalizeModelConfig(params.model); + agent.model = this.toRuntimeModel(normalizeModelConfig(params.model)); } if (params.enabled !== undefined) { this.upsertManagedAccountInConfig( @@ -645,8 +651,28 @@ export class AgentLifecycleService { return { agentId: record.id, name: agent.name, - model: agent.model, + model: this.toManagementModel(agent.model), enabled: params.enabled ?? record.enabled }; } + + private toRuntimeModel(model: { primary: string; fallbacks?: string[] }): { primary: string; fallbacks?: string[] } { + if (!this.env.modelGatewayAdminSocket) return model; + return { + primary: addGatewayPrefix(model.primary), + ...(Array.isArray(model.fallbacks) ? { fallbacks: model.fallbacks.map(addGatewayPrefix) } : {}) + }; + } + + private toManagementModel(model: unknown): unknown { + if (!this.env.modelGatewayAdminSocket || !model) return model; + const strip = (value: unknown) => String(value || "").replace(/^daedalus-gateway\//, ""); + if (typeof model === "string") return strip(model); + if (typeof model !== "object") return model; + const source = model as { primary?: unknown; fallbacks?: unknown[] }; + return { + primary: strip(source.primary), + ...(Array.isArray(source.fallbacks) ? { fallbacks: source.fallbacks.map(strip) } : {}) + }; + } } diff --git a/packages/daedalus-agent-management-web/server/management-src/capabilities/agent/service.ts b/packages/daedalus-agent-management-web/server/management-src/capabilities/agent/service.ts index 501ea13..6ab87af 100644 --- a/packages/daedalus-agent-management-web/server/management-src/capabilities/agent/service.ts +++ b/packages/daedalus-agent-management-web/server/management-src/capabilities/agent/service.ts @@ -48,6 +48,7 @@ import { AgentSkillsService } from "./skills-service.js"; import { resolveAgentCreateOptions, resolveBundledClawhubBin } from "./support.js"; import { AgentWorkspaceService } from "./workspace-service.js"; import { LlmModelConfigService } from "../llm/service.js"; +import { ModelGatewayAdminClient } from "../../tools/model-gateway-admin-client.js"; export class OpenClawManager { private readonly lifecycleService: AgentLifecycleService; @@ -76,7 +77,11 @@ export class OpenClawManager { doctorCli, env ); - this.llmModelConfigService = new LlmModelConfigService(env, this.operationsService); + this.llmModelConfigService = new LlmModelConfigService( + env, + this.operationsService, + env.modelGatewayAdminSocket ? new ModelGatewayAdminClient({ socketPath: env.modelGatewayAdminSocket }) : undefined + ); } async listAgents(): Promise { diff --git a/packages/daedalus-agent-management-web/server/management-src/capabilities/llm/service.ts b/packages/daedalus-agent-management-web/server/management-src/capabilities/llm/service.ts index 2e37ca0..a2b7151 100644 --- a/packages/daedalus-agent-management-web/server/management-src/capabilities/llm/service.ts +++ b/packages/daedalus-agent-management-web/server/management-src/capabilities/llm/service.ts @@ -16,6 +16,7 @@ import type { import { toChinaISOString } from "../../china-time.js"; import { AgentOperationsService } from "../agent/operations-service.js"; import { SHARED_FILE_MODE } from "../agent/support.js"; +import { ModelGatewayAdminClient } from "../../tools/model-gateway-admin-client.js"; const PROVIDER_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9_-]*$/; const MODEL_API = "openai-completions"; @@ -231,10 +232,14 @@ function collectModelRefs(model: unknown): string[] { export class LlmModelConfigService { constructor( private readonly env: EnvironmentConfig, - private readonly operationsService: AgentOperationsService + private readonly operationsService: AgentOperationsService, + private readonly gatewayClient?: ModelGatewayAdminClient ) {} async listProviders(): Promise<{ items: unknown[] }> { + if (this.gatewayClient) { + return await this.gatewayClient.call<{ items: unknown[] }>("providers.list"); + } const config = await this.readOpenClawConfig(); return { items: this.listProviderRecords(config) @@ -242,6 +247,16 @@ export class LlmModelConfigService { } async createProvider(params: LlmProviderCreateParams): Promise { + if (this.gatewayClient) { + const result = await this.gatewayClient.call("provider.create", { + providerId: params.providerId, + baseUrl: params.baseUrl, + apiKey: params.apiKey + }); + await this.syncGatewayOpenClawConfig(); + await this.operationsService.maybeRefreshGatewayAfterManagedTopologyChange(params.restart, `llm.provider.create:${params.providerId}`); + return result; + } const providerId = normalizeProviderId(params.providerId); const baseUrl = normalizeBaseUrl(params.baseUrl); const apiKey = normalizeApiKey(params.apiKey); @@ -270,6 +285,16 @@ export class LlmModelConfigService { } async updateProvider(params: LlmProviderUpdateParams): Promise { + if (this.gatewayClient) { + const result = await this.gatewayClient.call("provider.update", { + providerId: params.providerId, + ...(params.baseUrl !== undefined ? { baseUrl: params.baseUrl } : {}), + ...(params.apiKey !== undefined ? { apiKey: params.apiKey } : {}) + }); + await this.syncGatewayOpenClawConfig(); + await this.operationsService.maybeRefreshGatewayAfterManagedTopologyChange(params.restart, `llm.provider.update:${params.providerId}`); + return result; + } const providerId = normalizeProviderId(params.providerId); const config = await this.readOpenClawConfig(); const provider = this.requireProvider(config, providerId); @@ -289,6 +314,21 @@ export class LlmModelConfigService { } async deleteProvider(params: LlmProviderDeleteParams): Promise<{ providerId: string }> { + if (this.gatewayClient) { + const providerId = normalizeProviderId(params.providerId); + const config = await this.readOpenClawConfig(); + const usedRefs = this.findUsedModelRefs(config, (modelRef) => { + const normalized = modelRef.replace(/^daedalus-gateway\//, ""); + return normalized.startsWith(`${providerId}/`); + }); + if (usedRefs.length > 0) { + throw new Error(`LLM provider ${providerId} is used by agents: ${usedRefs.join(", ")}`); + } + const result = await this.gatewayClient.call<{ providerId: string }>("provider.delete", { providerId: params.providerId }); + await this.syncGatewayOpenClawConfig(); + await this.operationsService.maybeRefreshGatewayAfterManagedTopologyChange(params.restart, `llm.provider.delete:${params.providerId}`); + return result; + } const providerId = normalizeProviderId(params.providerId); const config = await this.readOpenClawConfig(); const provider = this.requireProvider(config, providerId); @@ -309,10 +349,25 @@ export class LlmModelConfigService { } async testProvider(params: LlmProviderTestParams): Promise { + if (this.gatewayClient) { + return await this.gatewayClient.call("provider.test", { + providerId: params.providerId, + ...(params.modelId ? { modelId: params.modelId } : {}), + ...(params.timeoutMs ? { timeoutMs: params.timeoutMs } : {}) + }); + } return await this.testModel(params); } async testModel(params: LlmModelTestParams): Promise { + if (this.gatewayClient) { + return await this.gatewayClient.call("model.test", { + ...(params.providerId ? { providerId: params.providerId } : {}), + ...(params.modelId ? { modelId: params.modelId } : {}), + ...(params.modelRef ? { modelRef: params.modelRef } : {}), + ...(params.timeoutMs ? { timeoutMs: params.timeoutMs } : {}) + }); + } const resolved = params.modelRef ? splitModelRef(params.modelRef) : { providerId: normalizeProviderId(params.providerId), modelId: params.modelId ? normalizeModelId(params.modelId) : "" @@ -461,6 +516,11 @@ export class LlmModelConfigService { } async listModels(params: { providerId?: string } = {}): Promise<{ items: unknown[] }> { + if (this.gatewayClient) { + return await this.gatewayClient.call<{ items: unknown[] }>("models.list", { + ...(params.providerId ? { providerId: params.providerId } : {}) + }); + } const config = await this.readOpenClawConfig(); const providers = this.listProviderRecords(config); const providerId = params.providerId ? normalizeProviderId(params.providerId) : ""; @@ -471,6 +531,18 @@ export class LlmModelConfigService { } async createModel(params: LlmModelCreateParams): Promise { + if (this.gatewayClient) { + const result = await this.gatewayClient.call("model.create", { + providerId: params.providerId, + modelId: params.modelId, + ...(params.name !== undefined ? { name: params.name } : {}), + ...(params.contextTokens !== undefined ? { contextTokens: params.contextTokens } : {}), + ...(params.maxTokens !== undefined ? { maxTokens: params.maxTokens } : {}) + }); + await this.syncGatewayOpenClawConfig(); + await this.operationsService.maybeRefreshGatewayAfterManagedTopologyChange(params.restart, `llm.model.create:${params.providerId}/${params.modelId}`); + return result; + } const providerId = normalizeProviderId(params.providerId); const modelId = normalizeModelId(params.modelId); const config = await this.readOpenClawConfig(); @@ -496,6 +568,19 @@ export class LlmModelConfigService { } async updateModel(params: LlmModelUpdateParams): Promise { + if (this.gatewayClient) { + const result = await this.gatewayClient.call("model.update", { + ...(params.providerId ? { providerId: params.providerId } : {}), + ...(params.modelId ? { modelId: params.modelId } : {}), + ...(params.modelRef ? { modelRef: params.modelRef } : {}), + ...(params.name !== undefined ? { name: params.name } : {}), + ...(params.contextTokens !== undefined ? { contextTokens: params.contextTokens } : {}), + ...(params.maxTokens !== undefined ? { maxTokens: params.maxTokens } : {}) + }); + await this.syncGatewayOpenClawConfig(); + await this.operationsService.maybeRefreshGatewayAfterManagedTopologyChange(params.restart, `llm.model.update:${params.modelRef || `${params.providerId}/${params.modelId}`}`); + return result; + } const resolved = params.modelRef ? splitModelRef(params.modelRef) : { providerId: normalizeProviderId(params.providerId), modelId: normalizeModelId(params.modelId) @@ -534,6 +619,28 @@ export class LlmModelConfigService { } async deleteModel(params: LlmModelDeleteParams): Promise<{ providerId: string; modelId: string; modelRef: string }> { + if (this.gatewayClient) { + const resolved = params.modelRef + ? splitModelRef(String(params.modelRef).replace(/^daedalus-gateway\//, "")) + : { + providerId: normalizeProviderId(params.providerId), + modelId: normalizeModelId(params.modelId) + }; + const modelRef = buildModelRef(resolved.providerId, resolved.modelId); + const config = await this.readOpenClawConfig(); + const usedRefs = this.findUsedModelRefs(config, (candidate) => candidate === modelRef || candidate === `daedalus-gateway/${modelRef}`); + if (usedRefs.length > 0) { + throw new Error(`LLM model ${modelRef} is used by agents: ${usedRefs.join(", ")}`); + } + const result = await this.gatewayClient.call<{ providerId: string; modelId: string; modelRef: string }>("model.delete", { + ...(params.providerId ? { providerId: params.providerId } : {}), + ...(params.modelId ? { modelId: params.modelId } : {}), + ...(params.modelRef ? { modelRef: params.modelRef } : {}) + }); + await this.syncGatewayOpenClawConfig(); + await this.operationsService.maybeRefreshGatewayAfterManagedTopologyChange(params.restart, `llm.model.delete:${params.modelRef || `${params.providerId}/${params.modelId}`}`); + return result; + } const resolved = params.modelRef ? splitModelRef(params.modelRef) : { providerId: normalizeProviderId(params.providerId), modelId: normalizeModelId(params.modelId) @@ -628,4 +735,31 @@ export class LlmModelConfigService { throw error; } } + + private async syncGatewayOpenClawConfig(): Promise { + if (!this.gatewayClient || !this.env.modelGatewayRuntimeToken) { + return; + } + const response = await this.gatewayClient.call<{ items?: Array> }>("models.list"); + const models = Array.isArray(response?.items) + ? response.items.map((model) => ({ + id: firstText(model.modelRef, model.modelId), + name: firstText(model.name, model.modelId), + ...(model.contextTokens != null ? { contextTokens: model.contextTokens } : {}), + ...(model.maxTokens != null ? { maxTokens: model.maxTokens } : {}) + })).filter((model) => model.id) + : []; + const config = await this.readOpenClawConfig(); + config.models ??= {}; + config.models.mode = "merge"; + config.models.providers = { + "daedalus-gateway": { + api: MODEL_API, + baseUrl: this.env.modelGatewayUrl || "http://127.0.0.1:3040/v1", + apiKey: this.env.modelGatewayRuntimeToken, + models + } + }; + await this.writeOpenClawConfig(config); + } } diff --git a/packages/daedalus-agent-management-web/server/management-src/env.ts b/packages/daedalus-agent-management-web/server/management-src/env.ts index a882312..1076260 100644 --- a/packages/daedalus-agent-management-web/server/management-src/env.ts +++ b/packages/daedalus-agent-management-web/server/management-src/env.ts @@ -97,6 +97,12 @@ export function loadEnvironmentConfig(): EnvironmentConfig { readOptionalPathEnv("DA_OPENCLAW_CLI_TMP_DIR", DEFAULT_OPENCLAW_CLI_TMP_DIR) ?? DEFAULT_OPENCLAW_CLI_TMP_DIR; const gatewayAuthToken = readRequiredEnv("DA_OPENCLAW_GATEWAY_AUTH_TOKEN", "daedalus-agent-local-gateway-token"); + const defaultModelGatewaySocket = readOptionalEnv("DA_RUN_MODE") === "container" + ? joinResolvedPath(dataRoot, "..", "..", "run", "model-gateway-admin.sock") + : undefined; + const modelGatewayAdminSocket = readOptionalPathEnv("DA_MODEL_GATEWAY_ADMIN_SOCKET", defaultModelGatewaySocket); + const modelGatewayRuntimeToken = readOptionalEnv("DA_MODEL_GATEWAY_RUNTIME_TOKEN"); + const modelGatewayUrl = readOptionalEnv("DA_MODEL_GATEWAY_URL", "http://127.0.0.1:3040/v1"); const managedBroker = readRequiredEnv("DA_MQTT_CHANNEL_BROKER"); const managedUsername = readRequiredEnv("DA_MQTT_CHANNEL_USERNAME"); @@ -113,6 +119,9 @@ export function loadEnvironmentConfig(): EnvironmentConfig { tmpDir, openclawCliTmpDir, gatewayAuthToken, + modelGatewayAdminSocket, + modelGatewayRuntimeToken, + modelGatewayUrl, openclawBin: readOptionalEnv("OPENCLAW_COMMAND", DEFAULT_OPENCLAW_COMMAND) ?? DEFAULT_OPENCLAW_BIN, doctorBin: readOptionalEnv("DOCTOR_COMMAND", DEFAULT_DOCTOR_COMMAND) ?? DEFAULT_DOCTOR_BIN, workspacesRoot: joinResolvedPath(openclawHome, "workspaces"), diff --git a/packages/daedalus-agent-management-web/server/management-src/tools/model-gateway-admin-client.ts b/packages/daedalus-agent-management-web/server/management-src/tools/model-gateway-admin-client.ts new file mode 100644 index 0000000..6ed9f0c --- /dev/null +++ b/packages/daedalus-agent-management-web/server/management-src/tools/model-gateway-admin-client.ts @@ -0,0 +1,54 @@ +import net from "node:net"; +import { randomUUID } from "node:crypto"; + +export interface ModelGatewayAdminClientOptions { + socketPath?: string; + timeoutMs?: number; +} + +export class ModelGatewayAdminClient { + private readonly socketPath: string; + private readonly timeoutMs: number; + + constructor(options: ModelGatewayAdminClientOptions = {}) { + this.socketPath = options.socketPath || process.env.DA_MODEL_GATEWAY_ADMIN_SOCKET || "/var/daedalus-agent/run/model-gateway-admin.sock"; + this.timeoutMs = Math.max(1000, Number(options.timeoutMs || 30000)); + } + + async call(action: string, params: Record = {}): Promise { + const id = randomUUID(); + return await new Promise((resolve, reject) => { + const socket = net.createConnection(this.socketPath); + let buffer = ""; + let settled = false; + const finish = (error?: Error, value?: T) => { + if (settled) return; + settled = true; + socket.destroy(); + if (error) reject(error); else resolve(value as T); + }; + const timer = setTimeout(() => finish(new Error(`Model Gateway admin request timed out: ${action}`)), this.timeoutMs); + socket.setEncoding("utf8"); + socket.on("connect", () => { + socket.write(`${JSON.stringify({ id, action, params })}\n`); + }); + socket.on("data", (chunk: string) => { + buffer += chunk; + const index = buffer.indexOf("\n"); + if (index < 0) return; + const line = buffer.slice(0, index); + clearTimeout(timer); + try { + const response = JSON.parse(line) as { id?: string; ok?: boolean; result?: T; error?: { message?: string } }; + if (response.id && response.id !== id) throw new Error("Model Gateway admin response id mismatch"); + if (!response.ok) throw new Error(response.error?.message || `Model Gateway admin action failed: ${action}`); + finish(undefined, response.result); + } catch (error) { + finish(error instanceof Error ? error : new Error(String(error))); + } + }); + socket.on("error", (error) => { clearTimeout(timer); finish(error); }); + socket.on("close", () => { clearTimeout(timer); if (!settled) finish(new Error(`Model Gateway admin socket closed: ${action}`)); }); + }); + } +} diff --git a/packages/daedalus-agent-management-web/server/management-src/types.ts b/packages/daedalus-agent-management-web/server/management-src/types.ts index 7f2cd97..904939b 100644 --- a/packages/daedalus-agent-management-web/server/management-src/types.ts +++ b/packages/daedalus-agent-management-web/server/management-src/types.ts @@ -14,6 +14,9 @@ export interface EnvironmentConfig { tmpDir: string; openclawCliTmpDir: string; gatewayAuthToken: string; + modelGatewayAdminSocket?: string; + modelGatewayRuntimeToken?: string; + modelGatewayUrl?: string; openclawBin: string; doctorBin: string; workspacesRoot: string; diff --git a/packages/daedalus-agent-management-web/server/management-test/llm-model-config-service.test.ts b/packages/daedalus-agent-management-web/server/management-test/llm-model-config-service.test.ts index 532ab9f..3e0f4b0 100644 --- a/packages/daedalus-agent-management-web/server/management-test/llm-model-config-service.test.ts +++ b/packages/daedalus-agent-management-web/server/management-test/llm-model-config-service.test.ts @@ -78,6 +78,31 @@ function createServiceWithOperations(env: EnvironmentConfig, calls: Array = []): LlmModelConfigService { + const operations = { + async maybeRefreshGatewayAfterManagedTopologyChange(restart: unknown, operation: string) { + calls.push({ restart, operation }); + } + } as unknown as AgentOperationsService; + const gatewayCalls: Array<{ action: string; params: Record }> = []; + const gatewayClient = { + async call(action: string, params: Record = {}): Promise { + gatewayCalls.push({ action, params }); + if (action === "provider.delete") return { providerId: params.providerId } as T; + if (action === "model.delete") return { + providerId: params.providerId || "provider", + modelId: params.modelId || "model", + modelRef: params.modelRef || `${params.providerId}/${params.modelId}` + } as T; + if (action === "models.list") return { items: [] } as T; + return {} as T; + } + } as any; + const service = new LlmModelConfigService(env, operations, gatewayClient); + (service as any).__gatewayCalls = gatewayCalls; + return service; +} + test("createModel treats empty token limits as unset", async () => { const root = await mkdtemp(path.join(tmpdir(), "oc-mgmt-llm-create-")); const env = await seedConfig(root); @@ -286,6 +311,27 @@ test("provider and model CRUD covers validation, filtering, and reference protec ]); }); +test("gateway provider and model deletion preserves agent reference protection", async () => { + const root = await mkdtemp(path.join(tmpdir(), "oc-mgmt-llm-gateway-delete-")); + const env = await seedConfig(root, []); + const config = await readConfig(env); + config.models.providers = { + "daedalus-gateway": { + api: "openai-completions", + baseUrl: "http://127.0.0.1:3040/v1", + apiKey: "runtime-token", + models: [{ id: "corp-openai/model" }] + } + }; + config.agents.list = [{ id: "agent-1", model: "daedalus-gateway/corp-openai/model" }]; + await writeFile(env.openclawConfigPath, `${JSON.stringify(config)}\n`, "utf8"); + const service = createGatewayService(env); + + await assert.rejects(service.deleteProvider({ providerId: "corp-openai" }), /used by agents/); + await assert.rejects(service.deleteModel({ providerId: "corp-openai", modelId: "model" }), /used by agents/); + assert.equal((service as any).__gatewayCalls.length, 0); +}); + test("model operations validate refs and preserve agent references", async () => { const root = await mkdtemp(path.join(tmpdir(), "oc-mgmt-llm-model-edge-")); const env = await seedConfig(root, [{ id: "existing", name: "Existing" }]); diff --git a/packages/daedalus-agent-model-gateway/package.json b/packages/daedalus-agent-model-gateway/package.json new file mode 100644 index 0000000..82bec1f --- /dev/null +++ b/packages/daedalus-agent-model-gateway/package.json @@ -0,0 +1,10 @@ +{ + "name": "daedalus-agent-model-gateway", + "version": "0.1.0", + "private": true, + "type": "module", + "scripts": { + "start": "node src/index.js", + "test": "node --test test/*.test.js" + } +} diff --git a/packages/daedalus-agent-model-gateway/src/index.js b/packages/daedalus-agent-model-gateway/src/index.js new file mode 100644 index 0000000..0541f3b --- /dev/null +++ b/packages/daedalus-agent-model-gateway/src/index.js @@ -0,0 +1,27 @@ +import process from "node:process"; +import { GatewayStore } from "./store.js"; +import { createGatewayServer } from "./server.js"; + +const env = process.env; +const root = env.DA_ROOT || "/var/daedalus-agent"; +const store = new GatewayStore({ + dataDir: env.DA_MODEL_GATEWAY_DATA_DIR || `${root}/model-gateway/data`, + secretDir: env.DA_MODEL_GATEWAY_SECRET_DIR || `${root}/model-gateway/secrets`, + masterKeyFile: env.DA_MODEL_GATEWAY_MASTER_KEY_FILE || `${root}/model-gateway/secrets/master.key`, + allowPrivateUpstream: ["1", "true", "yes"].includes(String(env.DA_MODEL_GATEWAY_ALLOW_PRIVATE_UPSTREAM || "").toLowerCase()) +}); +const gateway = createGatewayServer({ + store, + runtimeToken: env.DA_MODEL_GATEWAY_RUNTIME_TOKEN || "", + host: "127.0.0.1", + port: Number(env.DA_MODEL_GATEWAY_PORT || 3040), + adminSocket: env.DA_MODEL_GATEWAY_ADMIN_SOCKET || `${root}/run/model-gateway-admin.sock`, + usageEndpoint: env.DA_MODEL_GATEWAY_USAGE_ENDPOINT || `http://127.0.0.1:${env.DA_MANAGEMENT_CONSOLE_PORT || 3030}/api/internal/model-gateway/usage-events`, + usageToken: env.DA_MODEL_GATEWAY_USAGE_TOKEN || "", + upstreamTimeoutMs: Number(env.DA_MODEL_GATEWAY_UPSTREAM_TIMEOUT_MS || 120000) +}); + +await gateway.listen(); +console.log(`[model-gateway] ready on 127.0.0.1:${Number(env.DA_MODEL_GATEWAY_PORT || 3040)}`); + +for (const signal of ["SIGINT", "SIGTERM"]) process.on(signal, async () => { await gateway.close(); process.exit(0); }); diff --git a/packages/daedalus-agent-model-gateway/src/migrate.js b/packages/daedalus-agent-model-gateway/src/migrate.js new file mode 100644 index 0000000..148d502 --- /dev/null +++ b/packages/daedalus-agent-model-gateway/src/migrate.js @@ -0,0 +1,54 @@ +import fs from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import process from "node:process"; +import { GatewayStore } from "./store.js"; + +export function migrateConfig(configPath, store, logger = console) { + if (!configPath || !fs.existsSync(configPath)) return { migratedProviders: 0, migratedModels: 0 }; + + let config; + try { + config = JSON.parse(fs.readFileSync(configPath, "utf8").replace(/^\uFEFF/, "")); + } catch { + return { migratedProviders: 0, migratedModels: 0 }; + } + + const providers = config?.models?.providers; + if (!providers || typeof providers !== "object") return { migratedProviders: 0, migratedModels: 0 }; + + let migratedProviders = 0; + let migratedModels = 0; + for (const [providerId, provider] of Object.entries(providers)) { + if (providerId === "daedalus-gateway" || provider?.api !== "openai-completions" || !provider?.baseUrl || !provider?.apiKey) continue; + try { + const existing = store.listProviders().find((item) => item.providerId === providerId); + if (existing) continue; + store.createProvider({ providerId, baseUrl: provider.baseUrl, apiKey: provider.apiKey }); + migratedProviders += 1; + for (const model of Array.isArray(provider.models) ? provider.models : []) { + if (model?.id) { + store.createModel({ providerId, modelId: model.id, name: model.name, contextTokens: model.contextTokens, maxTokens: model.maxTokens }); + migratedModels += 1; + } + } + } catch (error) { + logger.error?.(`[model-gateway] legacy provider migration skipped for ${providerId}: ${error.message}`); + } + } + return { migratedProviders, migratedModels }; +} + +const scriptPath = fileURLToPath(import.meta.url); +if (process.argv[1] && path.resolve(process.argv[1]) === scriptPath) { + const configPath = process.argv[2]; + const env = process.env; + const root = env.DA_ROOT || "/var/daedalus-agent"; + const store = new GatewayStore({ + dataDir: env.DA_MODEL_GATEWAY_DATA_DIR || `${root}/model-gateway/data`, + secretDir: env.DA_MODEL_GATEWAY_SECRET_DIR || `${root}/model-gateway/secrets`, + masterKeyFile: env.DA_MODEL_GATEWAY_MASTER_KEY_FILE || `${root}/model-gateway/secrets/master.key`, + allowPrivateUpstream: ["1", "true", "yes"].includes(String(env.DA_MODEL_GATEWAY_ALLOW_PRIVATE_UPSTREAM || "").toLowerCase()) + }); + migrateConfig(configPath, store); +} diff --git a/packages/daedalus-agent-model-gateway/src/server.js b/packages/daedalus-agent-model-gateway/src/server.js new file mode 100644 index 0000000..9d1316d --- /dev/null +++ b/packages/daedalus-agent-model-gateway/src/server.js @@ -0,0 +1,237 @@ +import http from "node:http"; +import net from "node:net"; +import fs from "node:fs"; +import path from "node:path"; +import { randomUUID } from "node:crypto"; + +const MAX_BODY_BYTES = 5 * 1024 * 1024; + +function json(res, status, body, headers = {}) { + res.writeHead(status, { "Content-Type": "application/json; charset=utf-8", ...headers }); + res.end(JSON.stringify(body)); +} + +function readBody(req) { + return new Promise((resolve, reject) => { + let size = 0; + const chunks = []; + req.on("data", (chunk) => { + size += chunk.length; + if (size > MAX_BODY_BYTES) { + reject(Object.assign(new Error("request body too large"), { code: "request_too_large" })); + req.destroy(); + return; + } + chunks.push(chunk); + }); + req.on("end", () => resolve(Buffer.concat(chunks).toString("utf8"))); + req.on("error", reject); + }); +} + +function bearer(req) { + const value = String(req.headers.authorization || ""); + return value.startsWith("Bearer ") ? value.slice(7).trim() : ""; +} + +function errorStatus(error) { + if (error?.message === "gateway_auth_failed") return 401; + if (["model not found", "model alias is ambiguous"].some((prefix) => String(error?.message || "").startsWith(prefix))) return 404; + if (String(error?.message || "").includes("not allowed")) return 403; + if (error?.code === "request_too_large") return 413; + return 400; +} + +function modelAllowed(model, allowedModels) { + if (allowedModels.includes("*")) return true; + const normalized = String(model || "").replace(/^daedalus-gateway\//, ""); + return allowedModels.includes(model) || allowedModels.includes(normalized); +} + +function usageFromBody(body) { + const usage = body?.usage || {}; + return { + promptTokens: Number(usage.prompt_tokens || usage.promptTokens || 0), + completionTokens: Number(usage.completion_tokens || usage.completionTokens || 0), + totalTokens: Number(usage.total_tokens || usage.totalTokens || 0) + }; +} + +export function createGatewayServer({ store, runtimeToken = "", port = 3040, host = "127.0.0.1", adminSocket, usageEndpoint = "", usageToken = "", upstreamTimeoutMs = 120000, logger = console } = {}) { + if (!store) throw new Error("store is required"); + if (host !== "127.0.0.1") throw new Error("model gateway host must be 127.0.0.1"); + + async function recordUsage(event) { + if (!usageEndpoint) return; + try { + await fetch(usageEndpoint, { + method: "POST", + headers: { "Content-Type": "application/json", ...(usageToken ? { Authorization: `Bearer ${usageToken}` } : {}) }, + body: JSON.stringify(event), + signal: AbortSignal.timeout(5000) + }); + } catch (error) { + logger.warn?.({ error: error.message }, "model gateway usage event was not delivered"); + } + } + + async function proxy(req, res, auth, body, operation) { + const requestedModel = String(body.model || "").trim(); + if (!modelAllowed(requestedModel, auth.allowedModels || ["*"])) throw Object.assign(new Error("model not allowed"), { status: 403 }); + const resolved = store.resolveModel("", requestedModel); + const apiKey = store.getSecret(resolved.providerId); + const endpoint = `${resolved.provider.baseUrl.replace(/\/$/, "")}/${operation}`; + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), upstreamTimeoutMs); + const startedAt = Date.now(); + const requestId = String(req.headers["x-request-id"] || randomUUID()); + const upstreamBody = { ...body, model: resolved.modelId }; + try { + const upstream = await fetch(endpoint, { method: "POST", headers: { Authorization: `Bearer ${apiKey}`, "Content-Type": "application/json", Accept: body.stream ? "text/event-stream" : "application/json" }, body: JSON.stringify(upstreamBody), signal: controller.signal, redirect: "error" }); + if (!upstream.ok) { + const text = await upstream.text(); + let parsed; + try { parsed = JSON.parse(text); } catch { parsed = { message: text.slice(0, 2000) }; } + json(res, upstream.status >= 500 ? 502 : upstream.status, { error: { code: "upstream_error", message: parsed?.error?.message || parsed?.message || "upstream provider error", upstreamStatus: upstream.status } }); + await recordUsage({ eventId: requestId, requestId, agentId: auth.agentId || "runtime", senderId: "unknown", requestedModelRef: requestedModel, actualProviderId: resolved.providerId, actualModelRef: `${resolved.providerId}/${resolved.modelId}`, success: false, billable: false, errorCode: "upstream_error", latencyMs: Date.now() - startedAt }); + return; + } + if (body.stream) { + res.writeHead(upstream.status, { "Content-Type": upstream.headers.get("content-type") || "text/event-stream", "Cache-Control": "no-cache", Connection: "keep-alive" }); + const reader = upstream.body?.getReader(); + let pending = ""; + let usage = { promptTokens: 0, completionTokens: 0, totalTokens: 0 }; + while (reader) { + const next = await reader.read(); + if (next.done) break; + const chunk = Buffer.from(next.value); + res.write(chunk); + pending += chunk.toString("utf8"); + const lines = pending.split(/\r?\n/); + pending = lines.pop() || ""; + for (const line of lines) { + if (!line.startsWith("data:")) continue; + try { const parsed = JSON.parse(line.slice(5).trim()); if (parsed.usage) usage = usageFromBody(parsed); } catch { } + } + } + res.end(); + await recordUsage({ eventId: requestId, requestId, agentId: auth.agentId || "runtime", senderId: "unknown", requestedModelRef: requestedModel, actualProviderId: resolved.providerId, actualModelRef: `${resolved.providerId}/${resolved.modelId}`, ...usage, totalTokens: usage.totalTokens || usage.promptTokens + usage.completionTokens, latencyMs: Date.now() - startedAt }); + return; + } + const text = await upstream.text(); + let parsed; + try { parsed = text ? JSON.parse(text) : {}; } catch { parsed = { error: { code: "upstream_invalid_response", message: "upstream returned invalid JSON" } }; } + res.writeHead(upstream.status, { "Content-Type": upstream.headers.get("content-type") || "application/json" }); + res.end(JSON.stringify(parsed)); + const usage = usageFromBody(parsed); + await recordUsage({ eventId: requestId, requestId, agentId: auth.agentId || "runtime", senderId: "unknown", requestedModelRef: requestedModel, actualProviderId: resolved.providerId, actualModelRef: `${resolved.providerId}/${resolved.modelId}`, ...usage, totalTokens: usage.totalTokens || usage.promptTokens + usage.completionTokens, latencyMs: Date.now() - startedAt }); + } catch (error) { + const timeoutError = error?.name === "AbortError"; + if (!res.headersSent) json(res, 504, { error: { code: timeoutError ? "upstream_timeout" : "upstream_unavailable", message: timeoutError ? "upstream request timed out" : "upstream provider unavailable" } }); + await recordUsage({ eventId: requestId, requestId, agentId: auth.agentId || "runtime", senderId: "unknown", requestedModelRef: requestedModel, actualProviderId: resolved.providerId, actualModelRef: `${resolved.providerId}/${resolved.modelId}`, success: false, billable: false, errorCode: timeoutError ? "upstream_timeout" : "upstream_unavailable", latencyMs: Date.now() - startedAt }); + } finally { clearTimeout(timeout); } + } + + const httpServer = http.createServer(async (req, res) => { + try { + const url = new URL(req.url || "/", `http://${host}:${port}`); + if (req.method === "GET" && (url.pathname === "/healthz" || url.pathname === "/health/live")) return json(res, 200, { ok: true, service: "daedalus-model-gateway" }); + if (req.method === "GET" && (url.pathname === "/readyz" || url.pathname === "/health/ready")) return json(res, 200, { ok: true, service: "daedalus-model-gateway", providers: store.listProviders().length }); + let auth; + if (url.pathname.startsWith("/v1/")) auth = store.authorize(bearer(req), runtimeToken); + if (req.method === "GET" && url.pathname === "/v1/models") return json(res, 200, { object: "list", data: store.listModels().map((item) => ({ id: item.modelRef, object: "model", owned_by: item.providerId })) }); + if (req.method === "POST" && url.pathname === "/v1/chat/completions") { + const raw = await readBody(req); + let body; + try { body = JSON.parse(raw || "{}"); } catch { return json(res, 400, { error: { code: "invalid_json", message: "request body must be JSON" } }); } + return await proxy(req, res, auth, body, "chat/completions"); + } + if (req.method === "POST" && url.pathname === "/v1/embeddings") { + const raw = await readBody(req); + let body; + try { body = JSON.parse(raw || "{}"); } catch { return json(res, 400, { error: { code: "invalid_json", message: "request body must be JSON" } }); } + return await proxy(req, res, auth, body, "embeddings"); + } + return json(res, 404, { error: { code: "not_found", message: "route not found" } }); + } catch (error) { + const status = Number(error?.status) || errorStatus(error); + if (!res.headersSent) json(res, status, { error: { code: error?.message === "gateway_auth_failed" ? "gateway_auth_failed" : "gateway_error", message: status >= 500 ? "gateway request failed" : error.message } }); + } + }); + + let adminServer; + if (adminSocket) { + adminServer = net.createServer((socket) => { + let pending = ""; + socket.setEncoding("utf8"); + socket.on("data", async (chunk) => { + pending += chunk; + let index; + while ((index = pending.indexOf("\n")) >= 0) { + const line = pending.slice(0, index).trim(); + pending = pending.slice(index + 1); + if (!line) continue; + let request; + try { request = JSON.parse(line); } catch { socket.write(`${JSON.stringify({ ok: false, error: { code: "invalid_json", message: "invalid JSON" } })}\n`); continue; } + try { + const result = await dispatchAdmin(store, request); + socket.write(`${JSON.stringify({ id: request.id, ok: true, result })}\n`); + } catch (error) { + socket.write(`${JSON.stringify({ id: request.id, ok: false, error: { code: "admin_error", message: error.message } })}\n`); + } + } + }); + }); + } + + return { + httpServer, + adminServer, + async listen() { + if (adminServer) { + fs.mkdirSync(path.dirname(adminSocket), { recursive: true, mode: 0o770 }); + try { fs.unlinkSync(adminSocket); } catch (error) { if (error.code !== "ENOENT") throw error; } + await new Promise((resolve, reject) => { adminServer.once("error", reject); adminServer.listen(adminSocket, () => { try { fs.chmodSync(adminSocket, 0o660); } catch { } resolve(); }); }); + } + await new Promise((resolve, reject) => { httpServer.once("error", reject); httpServer.listen(port, host, resolve); }); + }, + async close() { await Promise.all([new Promise((resolve) => httpServer.close(() => resolve())), adminServer ? new Promise((resolve) => adminServer.close(() => resolve())) : Promise.resolve()]); try { if (adminSocket) fs.unlinkSync(adminSocket); } catch { } } + }; +} + +export async function dispatchAdmin(store, request) { + const action = String(request.action || request.method || ""); + const params = request.params || {}; + switch (action) { + case "status": return { ok: true, ready: true, providers: store.listProviders().length, models: store.listModels().length }; + case "providers.list": return { items: store.listProviders() }; + case "provider.create": return store.createProvider(params); + case "provider.update": return store.updateProvider(params); + case "provider.delete": return store.deleteProvider(params.providerId); + case "provider.test": return testProvider(store, params); + case "models.list": return { items: store.listModels(params.providerId || "") }; + case "model.create": return store.createModel(params); + case "model.update": return store.updateModel(params); + case "model.delete": return store.deleteModel(params); + case "model.test": return testProvider(store, params); + case "catalog.snapshot": return { llmProviders: { items: store.listProviders() } }; + case "client.issue": return store.issueClient(params); + case "client.revoke": return store.revokeClient(params.clientId); + case "policy.sync": return { ok: true, applied: true }; + default: throw new Error(`unsupported admin action: ${action}`); + } +} + +async function testProvider(store, params) { + let modelValue = params.modelId || params.modelRef; + if (!modelValue && params.providerId) { + const models = store.listModels(params.providerId); + if (!models.length) throw new Error(`provider ${params.providerId} has no models to test`); + modelValue = models[0].modelRef; + } + const resolved = store.resolveModel(params.providerId, modelValue); + const key = store.getSecret(resolved.providerId); + const response = await fetch(`${resolved.provider.baseUrl.replace(/\/$/, "")}/chat/completions`, { method: "POST", headers: { Authorization: `Bearer ${key}`, "Content-Type": "application/json" }, body: JSON.stringify({ model: resolved.modelId, messages: [{ role: "user", content: "Reply with exactly: DA_MODEL_TEST_OK" }], max_tokens: 16 }), signal: AbortSignal.timeout(Math.min(120000, Math.max(1000, Number(params.timeoutMs) || 30000))), redirect: "error" }); + const body = await response.text(); + return { ok: response.ok, providerId: resolved.providerId, modelId: resolved.modelId, modelRef: `${resolved.providerId}/${resolved.modelId}`, status: response.status, response: body ? (() => { try { return JSON.parse(body); } catch { return body.slice(0, 2000); } })() : null }; +} diff --git a/packages/daedalus-agent-model-gateway/src/store.js b/packages/daedalus-agent-model-gateway/src/store.js new file mode 100644 index 0000000..da85537 --- /dev/null +++ b/packages/daedalus-agent-model-gateway/src/store.js @@ -0,0 +1,290 @@ +import { createCipheriv, createDecipheriv, createHash, randomBytes, randomUUID, timingSafeEqual } from "node:crypto"; +import { chmodSync, existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from "node:fs"; +import path from "node:path"; +import net from "node:net"; + +const PROVIDER_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9_-]*$/; +const MODEL_ID_PATTERN = /^[^\s]{1,256}$/; +const PRIVATE_HOSTS = new Set(["localhost", "localhost.localdomain", "metadata.google.internal", "metadata"]); + +function text(value, field, { required = false, max = 512 } = {}) { + const result = String(value ?? "").trim(); + if (required && !result) throw new Error(`${field} is required`); + if (result.length > max) throw new Error(`${field} exceeds ${max} characters`); + return result; +} + +function validateProviderId(value) { + const result = text(value, "providerId", { required: true, max: 128 }); + if (!PROVIDER_ID_PATTERN.test(result)) throw new Error(`invalid providerId: ${result}`); + return result; +} + +function validateModelId(value) { + const result = text(value, "modelId", { required: true, max: 256 }); + if (!MODEL_ID_PATTERN.test(result)) throw new Error(`invalid modelId: ${result}`); + return result; +} + +function ipv4IsPrivate(hostname) { + const parts = hostname.split(".").map(Number); + if (parts.length !== 4 || parts.some((part) => !Number.isInteger(part) || part < 0 || part > 255)) return false; + return parts[0] === 10 + || parts[0] === 127 + || (parts[0] === 172 && parts[1] >= 16 && parts[1] <= 31) + || (parts[0] === 192 && parts[1] === 168) + || (parts[0] === 169 && parts[1] === 254) + || (parts[0] === 100 && parts[1] >= 64 && parts[1] <= 127); +} + +function validateBaseUrl(value, { allowPrivate = false } = {}) { + const raw = text(value, "baseUrl", { required: true, max: 2048 }).replace(/\/$/, ""); + let url; + try { + url = new URL(raw); + } catch { + throw new Error(`invalid baseUrl: ${raw}`); + } + if (!(["https:", "http:"].includes(url.protocol))) throw new Error("baseUrl must use http or https"); + if (url.username || url.password || url.search || url.hash) throw new Error("baseUrl cannot contain credentials, query, or fragment"); + const hostname = url.hostname.toLowerCase(); + if (!allowPrivate && (PRIVATE_HOSTS.has(hostname) || ipv4IsPrivate(hostname) || hostname === "[::1]" || hostname.startsWith("fc") || hostname.startsWith("fd"))) { + throw new Error("baseUrl cannot target loopback, private, link-local, or metadata hosts"); + } + return raw; +} + +function maskSecret(value) { + const secret = String(value || ""); + if (!secret) return ""; + return secret.length <= 10 ? `${secret.slice(0, 2)}****${secret.slice(-2)}` : `${secret.slice(0, 6)}****${secret.slice(-4)}`; +} + +function fingerprint(value) { + return createHash("sha256").update(String(value || "")).digest("hex"); +} + +function keyFromFile(filePath) { + mkdirSync(path.dirname(filePath), { recursive: true, mode: 0o700 }); + let raw; + if (existsSync(filePath)) raw = readFileSync(filePath); + else { + raw = randomBytes(32); + writeFileSync(filePath, raw, { mode: 0o600 }); + } + try { chmodSync(filePath, 0o600); } catch { /* Windows and read-only mounts */ } + if (raw.length === 32) return raw; + const value = raw.toString("utf8").trim(); + if (/^[0-9a-f]{64}$/i.test(value)) return Buffer.from(value, "hex"); + return createHash("sha256").update(raw).digest(); +} + +function encrypt(value, key) { + const iv = randomBytes(12); + const cipher = createCipheriv("aes-256-gcm", key, iv); + const ciphertext = Buffer.concat([cipher.update(String(value), "utf8"), cipher.final()]); + return { iv: iv.toString("base64url"), tag: cipher.getAuthTag().toString("base64url"), ciphertext: ciphertext.toString("base64url") }; +} + +function decrypt(record, key) { + const decipher = createDecipheriv("aes-256-gcm", key, Buffer.from(record.iv, "base64url")); + decipher.setAuthTag(Buffer.from(record.tag, "base64url")); + return Buffer.concat([decipher.update(Buffer.from(record.ciphertext, "base64url")), decipher.final()]).toString("utf8"); +} + +function atomicJson(filePath, value) { + const temp = `${filePath}.tmp-${process.pid}-${randomUUID()}`; + writeFileSync(temp, `${JSON.stringify(value, null, 2)}\n`, { mode: 0o600 }); + try { chmodSync(temp, 0o600); } catch { } + renameSync(temp, filePath); +} + +function publicModel(providerId, model) { + return { + providerId, + modelId: model.id, + modelRef: `${providerId}/${model.id}`, + name: model.name || model.id, + contextTokens: model.contextTokens ?? null, + maxTokens: model.maxTokens ?? null + }; +} + +function publicProvider(providerId, provider, secret) { + return { + providerId, + api: "openai-completions", + baseUrl: provider.baseUrl, + apiKeyConfigured: Boolean(provider.secret), + apiKeyMask: maskSecret(secret), + apiKeyFingerprint: provider.fingerprint || "", + models: (provider.models || []).map((model) => publicModel(providerId, model)) + }; +} + +export class GatewayStore { + constructor({ dataDir, secretDir, masterKeyFile, allowPrivateUpstream = false } = {}) { + this.dataDir = path.resolve(dataDir || path.join(process.cwd(), "data", "model-gateway")); + this.secretDir = path.resolve(secretDir || path.join(this.dataDir, "secrets")); + this.filePath = path.join(this.dataDir, "gateway.json"); + this.key = keyFromFile(path.resolve(masterKeyFile || path.join(this.secretDir, "master.key"))); + this.allowPrivateUpstream = allowPrivateUpstream; + mkdirSync(this.dataDir, { recursive: true, mode: 0o700 }); + mkdirSync(this.secretDir, { recursive: true, mode: 0o700 }); + this.state = this.read(); + } + + read() { + if (!existsSync(this.filePath)) return { version: 1, providers: {}, clients: {} }; + try { + const parsed = JSON.parse(readFileSync(this.filePath, "utf8")); + return { version: 1, providers: parsed.providers || {}, clients: parsed.clients || {} }; + } catch (error) { + throw new Error(`gateway store is invalid: ${error.message}`); + } + } + + save() { atomicJson(this.filePath, this.state); } + + listProviders() { + return Object.entries(this.state.providers).map(([id, provider]) => { + let secret = ""; + try { secret = provider.secret ? decrypt(provider.secret, this.key) : ""; } catch { } + return publicProvider(id, provider, secret); + }).sort((a, b) => a.providerId.localeCompare(b.providerId)); + } + + getProvider(providerId) { + const id = validateProviderId(providerId); + const provider = this.state.providers[id]; + if (!provider) throw new Error(`provider not found: ${id}`); + return { id, provider }; + } + + getSecret(providerId) { + const { provider } = this.getProvider(providerId); + if (!provider.secret) throw new Error(`provider ${providerId} has no apiKey`); + return decrypt(provider.secret, this.key); + } + + createProvider({ providerId, baseUrl, apiKey }) { + const id = validateProviderId(providerId); + if (this.state.providers[id]) throw new Error(`provider already exists: ${id}`); + const secret = text(apiKey, "apiKey", { required: true, max: 4096 }); + const url = validateBaseUrl(baseUrl, { allowPrivate: this.allowPrivateUpstream }); + this.state.providers[id] = { baseUrl: url, secret: encrypt(secret, this.key), fingerprint: fingerprint(secret), models: [] }; + this.save(); + return publicProvider(id, this.state.providers[id], secret); + } + + updateProvider({ providerId, baseUrl, apiKey }) { + const { id, provider } = this.getProvider(providerId); + if (baseUrl !== undefined) provider.baseUrl = validateBaseUrl(baseUrl, { allowPrivate: this.allowPrivateUpstream }); + let secret = this.getSecret(id); + if (apiKey !== undefined) { + secret = text(apiKey, "apiKey", { required: true, max: 4096 }); + provider.secret = encrypt(secret, this.key); + provider.fingerprint = fingerprint(secret); + } + this.save(); + return publicProvider(id, provider, secret); + } + + deleteProvider(providerId) { + const { id, provider } = this.getProvider(providerId); + if ((provider.models || []).length) throw new Error(`provider ${id} still has models`); + delete this.state.providers[id]; + this.save(); + return { providerId: id }; + } + + listModels(providerId = "") { + const providers = providerId ? [this.getProvider(providerId)] : Object.entries(this.state.providers).map(([id, provider]) => ({ id, provider })); + return providers.flatMap(({ id, provider }) => (provider.models || []).map((model) => publicModel(id, model))); + } + + createModel({ providerId, modelId, name, contextTokens, maxTokens }) { + const { id, provider } = this.getProvider(providerId); + const normalized = validateModelId(modelId); + if ((provider.models || []).some((item) => item.id === normalized)) throw new Error(`model already exists: ${id}/${normalized}`); + provider.models ||= []; + provider.models.push({ id: normalized, name: text(name || normalized, "name", { max: 512 }), ...(contextTokens != null ? { contextTokens: Number(contextTokens) } : {}), ...(maxTokens != null ? { maxTokens: Number(maxTokens) } : {}) }); + this.save(); + return publicModel(id, provider.models.at(-1)); + } + + updateModel({ providerId, modelId, modelRef, name, contextTokens, maxTokens }) { + const resolved = this.resolveModel(providerId, modelId || modelRef); + const model = resolved.provider.models.find((item) => item.id === resolved.modelId); + if (!model) throw new Error(`model not found: ${resolved.providerId}/${resolved.modelId}`); + if (name !== undefined) model.name = text(name || resolved.modelId, "name", { max: 512 }); + if (contextTokens !== undefined) model.contextTokens = contextTokens == null || contextTokens === "" ? undefined : Number(contextTokens); + if (maxTokens !== undefined) model.maxTokens = maxTokens == null || maxTokens === "" ? undefined : Number(maxTokens); + this.save(); + return publicModel(resolved.providerId, model); + } + + deleteModel({ providerId, modelId, modelRef }) { + const resolved = this.resolveModel(providerId, modelId || modelRef); + const before = resolved.provider.models.length; + resolved.provider.models = resolved.provider.models.filter((item) => item.id !== resolved.modelId); + if (before === resolved.provider.models.length) throw new Error(`model not found: ${resolved.providerId}/${resolved.modelId}`); + this.save(); + return { providerId: resolved.providerId, modelId: resolved.modelId, modelRef: `${resolved.providerId}/${resolved.modelId}` }; + } + + resolveModel(providerId, modelValue) { + const requested = text(modelValue, "model", { required: true, max: 512 }).replace(/^daedalus-gateway\//, ""); + if (requested.includes("/")) { + const slash = requested.indexOf("/"); + const id = validateProviderId(requested.slice(0, slash)); + const modelId = validateModelId(requested.slice(slash + 1)); + const { provider } = this.getProvider(id); + if (!(provider.models || []).some((model) => model.id === modelId)) throw new Error(`model not found: ${id}/${modelId}`); + return { providerId: id, provider, modelId }; + } + if (providerId) { + const id = validateProviderId(providerId); + const { provider } = this.getProvider(id); + const modelId = validateModelId(requested); + if (!(provider.models || []).some((model) => model.id === modelId)) throw new Error(`model not found: ${id}/${modelId}`); + return { providerId: id, provider, modelId }; + } + const matches = this.listModels().filter((item) => item.modelId === requested); + if (matches.length !== 1) throw new Error(matches.length ? `model alias is ambiguous: ${requested}` : `model not found: ${requested}`); + const { provider } = this.getProvider(matches[0].providerId); + return { providerId: matches[0].providerId, provider, modelId: matches[0].modelId }; + } + + issueClient({ agentId = "", allowedModels = ["*"], expiresInSeconds = 3600 } = {}) { + const token = `dgw_${randomBytes(32).toString("base64url")}`; + const clientId = randomUUID(); + const expiresAt = new Date(Date.now() + Math.max(60, Math.min(86400, Number(expiresInSeconds) || 3600)) * 1000).toISOString(); + this.state.clients[clientId] = { hash: fingerprint(token), agentId: text(agentId, "agentId", { max: 256 }), allowedModels: Array.isArray(allowedModels) ? allowedModels.map((item) => text(item, "allowedModels", { max: 512 })).filter(Boolean) : ["*"], expiresAt, revoked: false }; + this.save(); + return { clientId, token, agentId: this.state.clients[clientId].agentId, allowedModels: this.state.clients[clientId].allowedModels, expiresAt }; + } + + revokeClient(clientId) { + const id = text(clientId, "clientId", { required: true, max: 128 }); + if (!this.state.clients[id]) throw new Error(`client not found: ${id}`); + this.state.clients[id].revoked = true; + this.save(); + return { clientId: id, revoked: true }; + } + + authorize(token, runtimeToken = "") { + const value = String(token || ""); + if (runtimeToken && value === runtimeToken) return { agentId: "runtime", allowedModels: ["*"] }; + const hash = fingerprint(value); + for (const client of Object.values(this.state.clients)) { + if (client.revoked || !client.expiresAt || Date.parse(client.expiresAt) <= Date.now()) continue; + const left = Buffer.from(client.hash, "hex"); + const right = Buffer.from(hash, "hex"); + if (left.length === right.length && timingSafeEqual(left, right)) return { agentId: client.agentId || "runtime", allowedModels: client.allowedModels || ["*"] }; + } + throw new Error("gateway_auth_failed"); + } +} + +export { validateBaseUrl, maskSecret, fingerprint }; diff --git a/packages/daedalus-agent-model-gateway/test/migrate.test.js b/packages/daedalus-agent-model-gateway/test/migrate.test.js new file mode 100644 index 0000000..9b32763 --- /dev/null +++ b/packages/daedalus-agent-model-gateway/test/migrate.test.js @@ -0,0 +1,38 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { GatewayStore } from "../src/store.js"; +import { migrateConfig } from "../src/migrate.js"; + +test("migrates legacy providers from a runtime config into the encrypted gateway store", () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "daedalus-gateway-migrate-")); + const configPath = path.join(dir, "runtime-config.json"); + fs.writeFileSync(configPath, JSON.stringify({ + models: { + providers: { + legacy: { + api: "openai-completions", + baseUrl: "https://example.invalid/v1", + apiKey: "runtime-provider-secret", + models: [{ id: "model-a", name: "Model A", contextTokens: 8192 }] + } + } + } + })); + const store = new GatewayStore({ dataDir: path.join(dir, "data"), secretDir: path.join(dir, "secrets") }); + + const result = migrateConfig(configPath, store); + + assert.deepEqual(result, { migratedProviders: 1, migratedModels: 1 }); + assert.deepEqual(store.listModels(), [{ + providerId: "legacy", + modelId: "model-a", + modelRef: "legacy/model-a", + name: "Model A", + contextTokens: 8192, + maxTokens: null + }]); + assert.equal(fs.readFileSync(store.filePath, "utf8").includes("runtime-provider-secret"), false); +}); diff --git a/packages/daedalus-agent-model-gateway/test/server.test.js b/packages/daedalus-agent-model-gateway/test/server.test.js new file mode 100644 index 0000000..bbfcd94 --- /dev/null +++ b/packages/daedalus-agent-model-gateway/test/server.test.js @@ -0,0 +1,138 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import http from "node:http"; +import { GatewayStore } from "../src/store.js"; +import { createGatewayServer, dispatchAdmin } from "../src/server.js"; + +function tempStore() { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "daedalus-gateway-http-")); + return new GatewayStore({ dataDir: path.join(dir, "data"), secretDir: path.join(dir, "secrets"), allowPrivateUpstream: true }); +} + +test("requires gateway token and proxies OpenAI-compatible chat requests", async () => { + const upstream = http.createServer((req, res) => { + assert.equal(req.headers.authorization, "Bearer upstream-secret"); + res.setHeader("content-type", "application/json"); + res.end(JSON.stringify({ id: "chat-1", choices: [{ message: { role: "assistant", content: "ok" } }], usage: { prompt_tokens: 2, completion_tokens: 3, total_tokens: 5 } })); + }); + await new Promise((resolve) => upstream.listen(0, "127.0.0.1", resolve)); + const upstreamPort = upstream.address().port; + const usageEvents = []; + const usage = http.createServer((req, res) => { + let body = ""; + req.on("data", (chunk) => { body += chunk; }); + req.on("end", () => { usageEvents.push(JSON.parse(body)); res.end(JSON.stringify({ ok: true })); }); + }); + await new Promise((resolve) => usage.listen(0, "127.0.0.1", resolve)); + const store = tempStore(); + store.createProvider({ providerId: "gcapi", baseUrl: `http://127.0.0.1:${upstreamPort}/v1`, apiKey: "upstream-secret" }); + store.createModel({ providerId: "gcapi", modelId: "deepseek-v4-flash" }); + const gateway = createGatewayServer({ store, runtimeToken: "runtime-token", port: 0, adminSocket: "", usageEndpoint: `http://127.0.0.1:${usage.address().port}/events` }); + await gateway.listen(); + const port = gateway.httpServer.address().port; + const denied = await fetch(`http://127.0.0.1:${port}/v1/models`); + assert.equal(denied.status, 401); + const response = await fetch(`http://127.0.0.1:${port}/v1/chat/completions`, { method: "POST", headers: { Authorization: "Bearer runtime-token", "content-type": "application/json" }, body: JSON.stringify({ model: "gcapi/deepseek-v4-flash", messages: [{ role: "user", content: "hi" }] }) }); + assert.equal(response.status, 200); + assert.equal((await response.json()).choices[0].message.content, "ok"); + for (let attempt = 0; attempt < 20 && usageEvents.length === 0; attempt += 1) { + await new Promise((resolve) => setTimeout(resolve, 10)); + } + assert.equal(usageEvents.length, 1); + assert.equal(usageEvents[0].actualModelRef, "gcapi/deepseek-v4-flash"); + await gateway.close(); + await new Promise((resolve) => upstream.close(resolve)); + await new Promise((resolve) => usage.close(resolve)); +}); + +test("forwards SSE stream without changing event framing", async () => { + const upstream = http.createServer((_req, res) => { + res.writeHead(200, { "content-type": "text/event-stream" }); + res.write('data: {"choices":[{"delta":{"content":"a"}}]}\n\n'); + res.end('data: [DONE]\n\n'); + }); + await new Promise((resolve) => upstream.listen(0, "127.0.0.1", resolve)); + const store = tempStore(); + store.createProvider({ providerId: "p", baseUrl: `http://127.0.0.1:${upstream.address().port}/v1`, apiKey: "key" }); + store.createModel({ providerId: "p", modelId: "m" }); + const gateway = createGatewayServer({ store, runtimeToken: "token", port: 0, adminSocket: "" }); + await gateway.listen(); + const response = await fetch(`http://127.0.0.1:${gateway.httpServer.address().port}/v1/chat/completions`, { method: "POST", headers: { Authorization: "Bearer token", "content-type": "application/json" }, body: JSON.stringify({ model: "m", stream: true, messages: [] }) }); + assert.equal(response.headers.get("content-type"), "text/event-stream"); + assert.equal(await response.text(), 'data: {"choices":[{"delta":{"content":"a"}}]}\n\ndata: [DONE]\n\n'); + await gateway.close(); + await new Promise((resolve) => upstream.close(resolve)); +}); + +test("does not proxy models that are absent from the gateway catalog", async () => { + let upstreamCalls = 0; + const upstream = http.createServer((_req, res) => { + upstreamCalls += 1; + res.end(JSON.stringify({ choices: [] })); + }); + await new Promise((resolve) => upstream.listen(0, "127.0.0.1", resolve)); + const store = tempStore(); + store.createProvider({ providerId: "p", baseUrl: `http://127.0.0.1:${upstream.address().port}/v1`, apiKey: "key" }); + store.createModel({ providerId: "p", modelId: "known" }); + const gateway = createGatewayServer({ store, runtimeToken: "token", port: 0, adminSocket: "" }); + await gateway.listen(); + const response = await fetch(`http://127.0.0.1:${gateway.httpServer.address().port}/v1/chat/completions`, { method: "POST", headers: { Authorization: "Bearer token", "content-type": "application/json" }, body: JSON.stringify({ model: "p/unknown", messages: [] }) }); + assert.equal(response.status, 404); + assert.equal(upstreamCalls, 0); + await gateway.close(); + await new Promise((resolve) => upstream.close(resolve)); +}); + +test("proxies OpenAI-compatible embeddings through the configured provider", async () => { + const upstream = http.createServer((req, res) => { + assert.equal(req.url, "/v1/embeddings"); + let body = ""; + req.on("data", (chunk) => { body += chunk; }); + req.on("end", () => { + assert.equal(JSON.parse(body).model, "embedding-model"); + res.setHeader("content-type", "application/json"); + res.end(JSON.stringify({ object: "list", data: [{ object: "embedding", embedding: [0.1, 0.2], index: 0 }], usage: { prompt_tokens: 2, total_tokens: 2 } })); + }); + }); + await new Promise((resolve) => upstream.listen(0, "127.0.0.1", resolve)); + const store = tempStore(); + store.createProvider({ providerId: "p", baseUrl: `http://127.0.0.1:${upstream.address().port}/v1`, apiKey: "key" }); + store.createModel({ providerId: "p", modelId: "embedding-model" }); + const gateway = createGatewayServer({ store, runtimeToken: "token", port: 0, adminSocket: "" }); + await gateway.listen(); + const response = await fetch(`http://127.0.0.1:${gateway.httpServer.address().port}/v1/embeddings`, { method: "POST", headers: { Authorization: "Bearer token", "content-type": "application/json" }, body: JSON.stringify({ model: "p/embedding-model", input: "hello" }) }); + assert.equal(response.status, 200); + assert.deepEqual((await response.json()).data[0].embedding, [0.1, 0.2]); + await gateway.close(); + await new Promise((resolve) => upstream.close(resolve)); +}); + +test("only permits a loopback runtime binding", () => { + const store = tempStore(); + assert.throws(() => createGatewayServer({ store, host: "0.0.0.0" }), /127\.0\.0\.1/); +}); + +test("provider.test selects the first provider model when modelId is omitted", async () => { + const upstream = http.createServer((req, res) => { + let body = ""; + req.on("data", (chunk) => { body += chunk; }); + req.on("end", () => { + res.setHeader("x-upstream-authorization", req.headers.authorization || ""); + res.setHeader("x-upstream-model", JSON.parse(body).model); + res.setHeader("content-type", "application/json"); + res.end(JSON.stringify({ choices: [{ message: { content: "DA_MODEL_TEST_OK" } }] })); + }); + }); + await new Promise((resolve) => upstream.listen(0, "127.0.0.1", resolve)); + const store = tempStore(); + store.createProvider({ providerId: "p", baseUrl: `http://127.0.0.1:${upstream.address().port}/v1`, apiKey: "upstream-secret" }); + store.createModel({ providerId: "p", modelId: "first-model" }); + store.createModel({ providerId: "p", modelId: "second-model" }); + const result = await dispatchAdmin(store, { action: "provider.test", params: { providerId: "p" } }); + assert.equal(result.modelId, "first-model"); + assert.equal(result.ok, true); + await new Promise((resolve) => upstream.close(resolve)); +}); diff --git a/packages/daedalus-agent-model-gateway/test/store.test.js b/packages/daedalus-agent-model-gateway/test/store.test.js new file mode 100644 index 0000000..5b0d2a6 --- /dev/null +++ b/packages/daedalus-agent-model-gateway/test/store.test.js @@ -0,0 +1,37 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { GatewayStore } from "../src/store.js"; + +function createStore() { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "daedalus-gateway-")); + return new GatewayStore({ dataDir: path.join(dir, "data"), secretDir: path.join(dir, "secrets"), allowPrivateUpstream: true }); +} + +test("stores provider secrets encrypted and returns only masked metadata", () => { + const store = createStore(); + const provider = store.createProvider({ providerId: "gcapi", baseUrl: "https://gcapi.cn/v1", apiKey: "secret-provider-key" }); + assert.equal(provider.apiKeyMask, "secret****-key"); + assert.equal(store.listProviders()[0].apiKeyConfigured, true); + const raw = fs.readFileSync(store.filePath, "utf8"); + assert.equal(raw.includes("secret-provider-key"), false); + assert.equal(store.getSecret("gcapi"), "secret-provider-key"); +}); + +test("rejects private provider URLs by default", () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "daedalus-gateway-")); + const store = new GatewayStore({ dataDir: path.join(dir, "data"), secretDir: path.join(dir, "secrets") }); + assert.throws(() => store.createProvider({ providerId: "local", baseUrl: "http://127.0.0.1:9000", apiKey: "key" }), /private/); +}); + +test("issues and revokes scoped client tokens", () => { + const store = createStore(); + store.createProvider({ providerId: "p", baseUrl: "https://example.com/v1", apiKey: "key" }); + store.createModel({ providerId: "p", modelId: "m" }); + const issued = store.issueClient({ agentId: "agent-a", allowedModels: ["p/m"], expiresInSeconds: 3600 }); + assert.equal(store.authorize(issued.token).agentId, "agent-a"); + store.revokeClient(issued.clientId); + assert.throws(() => store.authorize(issued.token), /gateway_auth_failed/); +}); -- Gitee From 5ef28c98dd4e9687dcada8445ca117d270463a81 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=AE=81=E4=BC=9F?= Date: Wed, 2 Sep 2026 18:35:46 +0800 Subject: [PATCH 9/9] bugfix --- doc/system-feature-list.md | 29 +- .../enterprise-ai-gateway-refactoring-plan.md | 2 +- .../daedalus-agent-management-web/README.md | 14 +- .../docs/spec.md | 27 +- .../capabilities/agent/lifecycle-service.ts | 153 +------ .../capabilities/agent/operations-service.ts | 25 ++ .../capabilities/agent/service.ts | 4 - .../capabilities/agent/usage-list.ts | 5 - .../management-src/capabilities/context.ts | 1 - .../management-src/capabilities/registry.ts | 2 - .../server/management-src/service.ts | 1 - .../capabilities-registry.test.ts | 8 - .../management-test/coverage-contract.test.ts | 16 +- .../lifecycle-service-coverage.test.ts | 87 ---- .../management-test/openclaw-manager.test.ts | 105 ++--- .../management-test/service-queue.test.ts | 2 +- .../server/src/api/routes/index.js | 49 +-- .../server/src/app.js | 4 +- .../server/src/background/index.js | 27 -- .../server/src/docs/openapi.js | 6 +- .../server/src/services/agent-quota.js | 3 +- .../server/src/services/agent-service.js | 24 +- .../src/services/catalog-sync-service.js | 281 +------------ .../services/model-gateway-usage-service.js | 10 +- .../server/src/services/portal-service.js | 88 ++-- .../test/agent-service-alignment.test.js | 36 ++ .../server/test/app-smoke.test.js | 15 - .../server/test/background-services.test.js | 45 +- .../server/test/catalog-sync-route.test.js | 12 +- .../server/test/catalog-sync-service.test.js | 390 +----------------- .../test/management-action-client.test.js | 6 +- .../test/model-gateway-usage-route.test.js | 17 +- .../test/model-gateway-usage-service.test.js | 11 + .../test/portal-service-alignment.test.js | 56 ++- .../test/runtime-settings-route.test.js | 65 +++ .../src/app/App.jsx | 7 +- .../src/pages/SettingsPage.jsx | 106 +---- .../src/styles.css | 6 +- .../src/utils/startupSyncGate.js | 1 - .../src/utils/startupSyncGate.test.js | 2 +- .../test/run-browser-e2e.js | 4 +- .../src/server.js | 6 +- .../test/server.test.js | 11 + test/case/integration/playwright-flow.js | 6 +- 44 files changed, 430 insertions(+), 1345 deletions(-) delete mode 100644 packages/daedalus-agent-management-web/server/management-src/capabilities/agent/usage-list.ts diff --git a/doc/system-feature-list.md b/doc/system-feature-list.md index 709d3d3..cb56bd8 100644 --- a/doc/system-feature-list.md +++ b/doc/system-feature-list.md @@ -34,11 +34,11 @@ ### 2.2 总览、同步与版本 - [x][T] 仪表盘展示数字员工、模板、Skill、业务系统数量和 Token 用量等统计；“用量表（数字员工）”与“用量表（用户）”均按今日 Token 用量倒序、每页 10 条，仅展示今日有消耗的 Agent/用户，并直接聚合 Model Gateway 账本。 -- [x][T] 启动同步等待 `service.ping`，随后执行 `version.list`、`catalog.snapshot`；目录快照成功后延迟从 Model Gateway 用量账本刷新 Agent 快照。 -- [x][T] 目录和用量按计划自动刷新；每次完成后按配置间隔再次执行；页面可轮询同步状态。 -- [x][T] “同步数据”手动执行目录快照再执行用量刷新，并展示各对象的状态、开始/结束时间、错误和摘要。 -- [x] 将 Provider/Model、Agent、模板、Skill、业务系统、Ontology 目录和 Model Gateway 用量展示快照缓存到 SQLite；页面列表从 SQLite 读取而不是逐页直接调用远端管理动作。 -- [x][T] 缓存组件版本信息并在设置页展示 Gateway、插件、CLI 等版本状态。 +- [x][T] 启动同步等待 `service.ping`，随后执行 `version.list`、`catalog.snapshot`；目录快照按计划自动刷新。 +- [x][T] 目录按计划自动刷新；每次完成后按配置间隔再次执行；页面可轮询目录同步状态。 +- [x][T] “同步目录”手动执行 `catalog.snapshot`，展示各对象的状态、开始/结束时间、错误和摘要；不再执行用量刷新。 +- [x] 将 Provider/Model、Agent、模板、Skill、业务系统、Ontology 目录缓存到 SQLite；页面列表从 SQLite 读取而不是逐页直接调用远端管理动作。Model Gateway 用量仅保留 `model_usage_events` 账本并实时聚合。 +- [x][T] 缓存组件版本信息并在设置页展示 Gateway、插件、CLI 等版本状态；`/api/bootstrap` 返回前等待首次 `version.list` 刷新完成，确保系统设置页就绪时版本表数据完整。 ### 2.3 大语言模型管理 @@ -48,7 +48,7 @@ - [x][T] Model 连接测试，使用 OpenAI-compatible chat completion 验证配置和上下文。 - [x] 使用 `provider_id/model_id` 作为运行时模型引用；删除操作保护仍被 Agent/模板引用的记录。 - [x] 支持 Agent 主模型和备选模型；校验 Provider/Model 必须成对选择，备选模型不能与主模型相同。 -- [x][T] Model Gateway 用量账本 `model_usage_events` 按 `agentId`、`senderId` 记录请求、输入/输出/缓存/总 Token、实际模型、计费和幂等字段；管理端提供按 Agent、senderId、Agent+senderId、模型、天和小时的统计查询，并以网关账本作为今日配额和总览用量事实来源，保留兼容 Agent 快照。 +- [x][T] Model Gateway 用量账本 `model_usage_events` 按 `agentId`、`senderId` 记录请求、输入/输出/缓存/总 Token、实际模型、计费和幂等字段；Runtime Token 的 `agentId=runtime`、`senderId=unknown` 观测事件标记为不可计费，由 OpenClaw MQTT 插件的 `llm_output` 事件提供带用户维度的唯一计费记录，避免同一模型请求重复计量；管理端提供按 Agent、senderId、Agent+senderId、模型、天和小时的统计查询，并以网关账本作为 Agent 列表、今日配额和总览用量唯一事实来源；不再提供用量同步任务或界面，旧快照字段仅为兼容保留。 - [x][T] Model Gateway Runtime API 提供 `GET /healthz`、`GET /readyz`、`GET /v1/models`、`POST /v1/chat/completions` 和 `POST /v1/embeddings`，校验 Bearer Runtime/Client Token、模型白名单和固定路由，支持非流式与 SSE 流式 OpenAI 兼容转发；上游错误、超时和用量事件统一归一化。 ### 2.4 员工模板管理 @@ -77,7 +77,7 @@ - [x][T] Agent 新增、列表、详情、编辑、启用/停用、删除；ID/slug 校验并保持与远端 Agent 一致。 - [x][T] 配置名称、描述、标签、模板、主/备选 LLM、每日 Token 限额、状态和授权用户；概览 tab 在编辑状态下可分别修改主模型、备选模型和每日配额；用户范围支持指定用户或全部用户，全部用户模式不检查用户名是否存在于 Agent 平台用户列表，任何用户名均可使用。 - [x][T] 数字员工名称和编辑按钮均从列表进入全屏详情，操作列仅保留编辑、删除；详情包含只读概览、文件（默认加载 `AGENTS.md`）、技能、业务系统、MCP 和分配 tabs；技能、业务系统、MCP 三个 tab 复用员工模板对应的设置布局（模板式 `List`、`Segmented`、搜索、勾选、状态摘要和保存区域），数字员工技能页仅展示当前 Agent 专属 Skill，不展示全局或内置 Skill，分配复合框占满区块宽度，支持从现有用户选择或直接添加用户名；全部用户模式可使用任意用户名，指定用户模式按列表中的显式授权生效。 -- [x][T] 状态包括 `normal`、`disabled`、`overlimit`；Model Gateway 用量快照刷新完成后，Agent 列表与已打开的详情概览同步展示最新 Token 用量及当天 sender 明细；超过每日限额时自动标记 `overlimit`，不可直接恢复为 `normal` 直到超额问题解决。 +- [x][T] 状态包括 `normal`、`disabled`、`overlimit`；Agent 列表与已打开的详情概览直接展示 Model Gateway 账本的最新 Token 用量及当天 sender 明细；超过每日限额时自动标记 `overlimit`，不可直接恢复为 `normal` 直到超额问题解决。 - [x][T] 创建/更新具备远端先行、本地 SQLite 落库、失败补偿和回滚；远端已不存在的 Agent 可清理本地记录。 - [x][T] Agent 用户目录支持分页搜索、单个新增、换行批量导入、删除和用量/关联员工查询；用户可配置每日 Token 限额（默认 `-1` 不限额），列表展示每日限额、今日用量和状态，支持修改用户名/限额；“已分配的数字员工”仅展示显式授权项，不包含访问策略为“全部用户”的 Agent。 - [x][T] 为 Agent 分配或移除用户，支持带策略版本的“全部用户/指定用户”访问范围；全部用户模式不检查 `userName` 是否存在于 `agent_users`，外部目录和外部会话均允许任何用户名使用，首次创建会话时自动登记该用户名；指定用户模式仍要求显式授权；未传身份时仅返回全部用户可用的 Agent，并在每个目录项返回 `is_public` 标记。 @@ -87,7 +87,7 @@ - [x][T] Agent 工作区 Markdown 文件列表、读取、创建、保存、移动/重命名、删除、下载，使用 etag 做乐观并发校验；文件页不展示额外的“新会话生效”提示。 - [x][T] 恢复 Agent 默认 `AGENTS.md`；Agent Skill 列表、ZIP 安装/替换、删除、导出，支持 etag 和 slug 校验。 - [x][T] 管理控制台提供 Agent 入站 Topic 探测、管理 Topic 探测和对话测试；对话测试展示发送消息、回复消息、步骤耗时和完整 MQTT 报文，并在数字员工达到每日配额时拒绝测试请求。 -- [x][T] 为每个 Agent 暴露同步 HTTP 调用 URL；`POST /api/external/agents/{agentId}/call` 接收 `senderId`、`text` 和可选 `cookie`，若 `senderId` 不在非管理员 Agent 用户列表 `agent_users` 中则以默认每日限额 `-1`（不限额）幂等创建该用户（不写入管理员 `users` 表）；服务端在调用前校验用户和 Agent 的今日网关用量，任一达到每日限额即拒绝并明确提示超限主体；服务端为每次请求生成一次性 MQTT 关联会话 ID，在进程内注册 Agent、sender 和 Cookie，通过 MQTT 请求 Agent 并固定只返回匹配临时会话的 `kind=final` 最终结果（发送 `disableBlockStreaming: true`，忽略中间 `block`）；传入 Cookie 仅在本次临时会话调用业务系统时使用，不写入 MQTT 报文；临时会话使用后废弃，临时 session ID、Cookie 不写入 `external_sessions`/`external_session_cookies` 或调用日志 `session_id`；成功后刷新 Agent 用量快照，将本次调用产生的 Token 计入每日配额并更新超额状态。 +- [x][T] 为每个 Agent 暴露同步 HTTP 调用 URL；`POST /api/external/agents/{agentId}/call` 接收 `senderId`、`text` 和可选 `cookie`，若 `senderId` 不在非管理员 Agent 用户列表 `agent_users` 中则以默认每日限额 `-1`（不限额）幂等创建该用户（不写入管理员 `users` 表）；服务端在调用前校验用户和 Agent 的今日网关用量，任一达到每日限额即拒绝并明确提示超限主体；服务端为每次请求生成一次性 MQTT 关联会话 ID，在进程内注册 Agent、sender 和 Cookie，通过 MQTT 请求 Agent 并固定只返回匹配临时会话的 `kind=final` 最终结果（发送 `disableBlockStreaming: true`，忽略中间 `block`）；传入 Cookie 仅在本次临时会话调用业务系统时使用，不写入 MQTT 报文；临时会话使用后废弃，临时 session ID、Cookie 不写入 `external_sessions`/`external_session_cookies` 或调用日志 `session_id`；Model Gateway 写入的事件直接计入每日配额。 ### 2.7 MCP 服务管理 @@ -114,7 +114,7 @@ - [x][T] 系统日志下载支持按日期选择 MQTT、MinIO、内核、应用、OpenClaw、管理端、invoke service、MQTT 通讯和交互日志。 - [x][T] 基础设置维护门户信息、对话/业务应用服务器地址、运行凭据和 IP 白名单；运行凭据支持重置 MQTT/MinIO 密码，重置会请求容器重启以使新密码生效。 - [x][T] 设置页展示 MQTT/MinIO/服务运行健康状态，支持刷新健康缓存；运行凭据区分“重置密码”和“刷新状态”操作，并在重置前要求确认。 -- [x][T] 支持目录同步、查询 Gateway 状态、重启 Gateway、重启容器、执行任意已注册管理指令和运行服务器诊断。 +- [x][T] 支持目录同步、查询 Gateway 状态、重启 Gateway、重启容器、执行任意已注册管理指令和运行服务器诊断；管理端收到 OpenClaw `scheduled`、`coalesced` 或 `deferred` 重启结果时，会等待排队重启实际执行并在 Gateway 恢复就绪后再返回。 - [T] 设置页对重启进行轮询等待、提示服务重启状态；后台实时更新通过 WebSocket/update notifier 推送。 ## 3. 外部与内部 API @@ -123,7 +123,7 @@ - [x][T] 管理 Web 提供面向 `/api/external/*` 对外接口的 OpenAPI 3.0 文档和本地 Swagger UI：文档入口为 `/doc`，规范 JSON 为 `/doc/openapi.json`，包含应用场景、鉴权、参数用途/限制/可选项、请求体和常见错误说明；基础设置的版本信息下方提供“API 文档”入口。 - [x][T] 持久外部会话 ID 使用 `s-` 加 30 位数字格式；同一用户-Agent 复用会话；会话创建校验 Agent 状态和用户授权。同步调用使用独立的一次性 MQTT 关联 ID，不属于持久外部会话。 - [x][T] 外部调用在每次解析业务系统时重新校验会话有效性、Agent 状态、业务系统状态和 Agent 业务系统策略。 -- [x][T] 内部 CUI API：查询可用 Agent 目录；内部 invoke API：解析授权业务系统、获取业务系统 base URL、记录调用日志；Model Gateway 用量内部 API 写入幂等事件并提供每日配额检查；OpenClaw MQTT 插件通过 `llm_output` 落账并在 `before_agent_run` 阻断超限请求，提示明确指出超限用户或数字员工。 +- [x][T] 内部 CUI API：查询可用 Agent 目录；内部 invoke API：解析授权业务系统、获取业务系统 base URL、记录调用日志；Model Gateway 用量内部 API 写入幂等事件并提供每日配额检查，Runtime Token 的无用户观测事件不计入账本统计；OpenClaw MQTT 插件通过 `llm_output` 落账并在 `before_agent_run` 阻断超限请求，提示明确指出超限用户或数字员工。 - [x][T] OpenClaw 配置迁移后只保留 `daedalus-gateway` Provider、`http://127.0.0.1:3040/v1` 和 opaque Runtime Token；启动时将旧 OpenAI Provider 密钥迁移到网关加密存储并删除直连 Provider 配置；Agent 出口默认阻断外部 TCP `80/443` 直连模型 Provider。 - [x][T] 管理员用量 API：`GET /api/usage/summary` 支持按 Agent、senderId、Agent+senderId、模型和时间窗口聚合；`GET /api/usage/agents/{agentId}/senders` 查询指定 Agent 的 senderId 明细。 - [x][T] API 鉴权分层：管理端登录 Token、服务 Bearer Token、内部请求校验；localhost 请求按配置可走内部通道。 @@ -170,18 +170,19 @@ | 范围 | 主要验证内容 | 代表测试位置 | | --- | --- | --- | -| 管理 Web 服务 | 路由、鉴权、用户/Agent/模板/Skill/LLM/MCP/业务系统 CRUD、内置模板编辑/重命名与删除保护、模板主/备选模型和 Agent 继承、Agent 用户访问策略（全部用户模式允许目录外未知用户名、指定用户模式仍需显式授权）、用户每日配额维护及今日用量/状态、用户改名与网关事件事务更新、Agent 多实例 MCP 选定服务同步及普通/沙箱 deny 投影、外部 Agent 目录匿名公开筛选与 `is_public` 标记、同步 Agent HTTP 接口（参数校验、服务鉴权、可选 Cookie、首次同步调用自动创建非管理员 Agent 用户、用户/Agent 任一配额超限时明确拒绝、临时会话业务系统调用、MQTT final-only 行为、临时 session 清理、成功后刷新用量并计入每日配额）、管理端 Agent 对话测试配额拦截、Model Gateway Provider/Model Admin Socket 调用、Model Gateway 用量事件幂等写入、按 Agent/senderId 聚合查询、Agent sender 明细、运行凭据密码重置及容器重启请求、OpenAPI/Swagger 文档外部接口和用量接口、审计、同步、健康检查、日志下载、SQLite 回滚和制品安全 | `packages/daedalus-agent-management-web/server/test/app-smoke.test.js`、`packages/daedalus-agent-management-web/server/test/model-gateway-usage-service.test.js`、`packages/daedalus-agent-management-web/server/test/model-gateway-usage-route.test.js`、`packages/daedalus-agent-management-web/server/test/catalog-sync-service.test.js`、`packages/daedalus-agent-management-web/server/test/db-reset-migration.test.js`、`packages/daedalus-agent-management-web/server/test/user-quota.test.js`、`packages/daedalus-agent-management-web/server/test/portal-service-alignment.test.js`、`packages/daedalus-agent-management-web/server/test/runtime-env-config-service.test.js`、`packages/daedalus-agent-management-web/server/test/runtime-settings-route.test.js`、`packages/daedalus-agent-management-web/server/test/topic-ping-service.test.js`、`packages/daedalus-agent-management-web/server/test/agent-user-policy.test.js`、`packages/daedalus-agent-management-web/server/test/external-service.test.js`、`packages/daedalus-agent-management-web/server/test/agent-service-alignment.test.js`、`packages/daedalus-agent-management-web/server/test/env-config.test.js`、`packages/daedalus-agent-management-web/server/test/mcp-service.test.js`、`packages/daedalus-agent-management-web/server/management-test/mcp-sync.test.ts`、`packages/daedalus-agent-model-gateway/test/store.test.js`、`packages/daedalus-agent-model-gateway/test/server.test.js`、`packages/daedalus-agent-model-gateway/test/migrate.test.js`、其他 `server/test/`、`server/management-test/` | -| 管理 Web 前端 | API 客户端鉴权/超时/错误归一化、同步门控、重启轮询、格式化、浏览器覆盖契约、员工与模板列表操作精简、内置模板编辑、名称/编辑详情入口、详情 tabs 保存流程、员工概览主模型/备选模型/每日配额快捷修改及打开状态下的 Token 用量同步刷新、用户每日限额/今日用量/状态列表、用户新增/编辑/删除、Dashboard 用户/Agent 用量表 10 条分页与零用量过滤、员工技能/业务系统/MCP 与模板设置布局一致及内置 Skill 隐藏、编辑页不展示同步调用 URL、模板文本文件切换后的编辑器可见性、设置页 API 文档入口、运行凭据重置按钮/确认框及“刷新状态”按钮 | `packages/daedalus-agent-management-web/src/**/*.test.js`、`packages/daedalus-agent-management-web/src/pages/AgentsPage.test.jsx`、`packages/daedalus-agent-management-web/src/pages/AgentUsersPage.test.jsx`、`packages/daedalus-agent-management-web/src/pages/DashboardPage.test.jsx`、`packages/daedalus-agent-management-web/test/browser-coverage.test.js`、`packages/daedalus-agent-management-web/test/run-browser-e2e.js`、`packages/daedalus-agent-management-web/test/integration/website-full.js` | +| Model Gateway 用量计费去重 | Runtime Token 的 `agentId=runtime`、`senderId=unknown` 观测事件标记为不可计费，并从 Dashboard、配额和聚合查询中排除；带 Agent 身份的 Client Token 仍计费 | `packages/daedalus-agent-model-gateway/test/server.test.js`、`packages/daedalus-agent-management-web/server/test/model-gateway-usage-service.test.js`、`packages/daedalus-agent-management-web/server/test/portal-service-alignment.test.js` | +| 管理 Web 服务 | 路由、鉴权、用户/Agent/模板/Skill/LLM/MCP/业务系统 CRUD、内置模板编辑/重命名与删除保护、模板主/备选模型和 Agent 继承、Agent 用户访问策略（全部用户模式允许目录外未知用户名、指定用户模式仍需显式授权）、用户每日配额维护及今日用量/状态、用户改名与网关事件事务更新、Agent 多实例 MCP 选定服务同步及普通/沙箱 deny 投影、外部 Agent 目录匿名公开筛选与 `is_public` 标记、同步 Agent HTTP 接口（参数校验、服务鉴权、可选 Cookie、首次同步调用自动创建非管理员 Agent 用户、用户/Agent 任一配额超限时明确拒绝、临时会话业务系统调用、MQTT final-only 行为、临时会话清理、Model Gateway 事件计入每日配额）、管理端 Agent 对话测试配额拦截、Model Gateway Provider/Model Admin Socket 调用、Model Gateway 用量事件幂等写入、按 Agent/senderId 聚合查询、Agent sender 明细、目录同步仅调用 `catalog.snapshot`、运行凭据密码重置及容器重启请求、OpenAPI/Swagger 文档外部接口和用量接口、审计、同步、健康检查、日志下载、SQLite 回滚和制品安全 | `packages/daedalus-agent-management-web/server/test/app-smoke.test.js`、`packages/daedalus-agent-management-web/server/test/model-gateway-usage-service.test.js`、`packages/daedalus-agent-management-web/server/test/model-gateway-usage-route.test.js`、`packages/daedalus-agent-management-web/server/test/catalog-sync-service.test.js`、`packages/daedalus-agent-management-web/server/test/db-reset-migration.test.js`、`packages/daedalus-agent-management-web/server/test/user-quota.test.js`、`packages/daedalus-agent-management-web/server/test/portal-service-alignment.test.js`、`packages/daedalus-agent-management-web/server/test/runtime-env-config-service.test.js`、`packages/daedalus-agent-management-web/server/test/runtime-settings-route.test.js`、`packages/daedalus-agent-management-web/server/test/topic-ping-service.test.js`、`packages/daedalus-agent-management-web/server/test/agent-user-policy.test.js`、`packages/daedalus-agent-management-web/server/test/external-service.test.js`、`packages/daedalus-agent-management-web/server/test/agent-service-alignment.test.js`、`packages/daedalus-agent-management-web/server/test/env-config.test.js`、`packages/daedalus-agent-management-web/server/test/mcp-service.test.js`、`packages/daedalus-agent-management-web/server/management-test/mcp-sync.test.ts`、`packages/daedalus-agent-model-gateway/test/store.test.js`、`packages/daedalus-agent-model-gateway/test/server.test.js`、`packages/daedalus-agent-model-gateway/test/migrate.test.js`、其他 `server/test/`、`server/management-test/` | +| 管理 Web 前端 | API 客户端鉴权/超时/错误归一化、目录同步门控、重启轮询、格式化、浏览器覆盖契约、员工与模板列表操作精简、内置模板编辑、名称/编辑详情入口、详情 tabs 保存流程、员工概览主模型/备选模型/每日配额快捷修改及账本用量刷新、用户每日限额/今日用量/状态列表、用户新增/编辑/删除、Dashboard 用户/Agent 用量表 10 条分页与零用量过滤、员工技能/业务系统/MCP 与模板设置布局一致及内置 Skill 隐藏、编辑页不展示同步调用 URL、模板文本文件切换后的编辑器可见性、设置页 API 文档入口、运行凭据重置按钮/确认框及“刷新状态”按钮、Bootstrap 等待版本缓存后再展示设置页版本表 | `packages/daedalus-agent-management-web/src/**/*.test.js`、`packages/daedalus-agent-management-web/src/pages/AgentsPage.test.jsx`、`packages/daedalus-agent-management-web/src/pages/AgentUsersPage.test.jsx`、`packages/daedalus-agent-management-web/src/pages/DashboardPage.test.jsx`、`packages/daedalus-agent-management-web/server/test/runtime-settings-route.test.js`、`packages/daedalus-agent-management-web/server/test/portal-service-alignment.test.js`、`packages/daedalus-agent-management-web/test/browser-coverage.test.js`、`packages/daedalus-agent-management-web/test/run-browser-e2e.js`、`packages/daedalus-agent-management-web/test/integration/website-full.js` | | invoke CLI | 参数、JSON/body/query 来源、Binding schema、socket 协议、超时、大小限制、Provider 错误和调用日志 | `packages/daedalus-agent-apps-invoke-cli/test/` | | MQTT 插件 | 账号配置、旧配置兼容、环境变量覆盖、连接/断线重连、通配符、入站路由、普通消息串行队列、停止与 steer/tell 控制队列、active-run 非阻塞 steer、运行时接收 Promise 未完成时的确认与原 final 投递、原始会话目标、OpenClaw deliver 的 text/kind 无加工透传、出站 envelope、日志、Model Gateway `llm_output` 用量落账、`before_agent_run` 配额阻断和待写入同步 | `extensions/openclaw-plugins/daedalus-agent-mqtt-channel/src/*test.ts`、`extensions/openclaw-plugins/daedalus-agent-mqtt-channel/index.test.ts` | -| Docker 集成 | 登录、版本/同步、Provider/Model/模板/MCP/Agent 生命周期、模板继承、工作区/Skill、Ontology、MQTT `/stop`/`/steer`/`/tell` 的 final 确认及普通队列保留、重启、启动钩子、包持久化、文件输出、会话隔离和压测 | `test/case/integration/`、`test/suite/integration/`、`test/case/stress/` | +| Docker 集成 | 登录、版本/同步、Provider/Model/模板/MCP/Agent 生命周期、模板继承、工作区/Skill、Ontology、MQTT `/stop`/`/steer`/`/tell` 的 final 确认及普通队列保留、重启（含排队重启等待和配置生效）、启动钩子、包持久化、文件输出、会话隔离和压测；指定镜像使用真实模型 Gateway 路径验证 Agent 实际调用 `integration-mcp__employee_template_lookup` MCP 工具，并验证 MCP deny 投影 | `test/case/integration/`、`test/suite/integration/`、`test/case/stress/`；报告 `E:\tmp\daedalus-agent-it-b2f7-fix\reports\playwright-container-flow-2026-09-02T10-07-12-545Z.json` | | Docker 升级与品牌契约 | OpenClaw/Node 版本、插件 seed、迁移、重启保护、持久化 MQTT 账号密码与运行凭据同步、DA_ 环境变量前缀和视觉品牌 | `test/case/upgrade/openclaw-upgrade-contract.test.js`、`test/case/upgrade/`、`test/case/branding/` | ## 8. 当前边界与使用前提 - [!] 管理控制台是管理员/运维面板，不是终端对话应用；终端用户通过 CUI 或 MQTT 接入 Agent。 - [!] 用户和数字员工每日限额均以 Model Gateway `model_usage_events` 中中国自然日、`billable = 1` 的总 Token 为准；任一达到限额后，在下一个中国自然日开始前禁止外部会话、已有会话上下文获取、同步 Agent 调用、管理端 Agent 对话测试和业务系统解析，错误会明确指出超限用户或数字员工；`-1` 表示不限额。 -- [!] 同步 Agent HTTP 接口是服务间调用接口，必须使用服务 Bearer Token（本机请求按现有服务鉴权策略处理）；它只等待并返回最终结果，成功后刷新 Agent 用量以计入每日配额，调用方仍需要自行处理 MQTT/Agent 超时或服务不可用错误。 +- [!] 同步 Agent HTTP 接口是服务间调用接口，必须使用服务 Bearer Token（本机请求按现有服务鉴权策略处理）；它只等待并返回最终结果，Model Gateway 用量事件直接计入每日配额，调用方仍需要自行处理 MQTT/Agent 超时或服务不可用错误。 - [!] 业务系统当前按 HZG/活字格模型接入，Ontology ZIP 和 `index.md` 是必要前提。 - [!] Agent 只能访问配置白名单内的内网地址；白名单更新后必须重启容器。 - [!] 内置模板、内置 Skill、内置 Ontology 受删除保护；内置 Skill 和内置 Ontology 不能按普通资源直接编辑。 diff --git a/docker/docs/enterprise-ai-gateway-refactoring-plan.md b/docker/docs/enterprise-ai-gateway-refactoring-plan.md index 5073c5d..e5c67e5 100644 --- a/docker/docs/enterprise-ai-gateway-refactoring-plan.md +++ b/docker/docs/enterprise-ai-gateway-refactoring-plan.md @@ -443,7 +443,7 @@ Gateway Client Key 必须独立于管理 Token，服务端只保存哈希。MVP ### 6.8 用量账本 MVP 落地 -当前管理端已先落地 Model Gateway 用量账本的最小闭环：Gateway 在模型请求完成后向管理端内部接口写入 `model_usage_events`，事件要求包含 `agentId` 和 `senderId`，并用 `eventId`/`requestId` 幂等去重。管理端提供 `GET /api/usage/summary` 和 `GET /api/usage/agents/{agentId}/senders` 查询接口，并只将日/周/月/总量及当天 sender 明细缓存到 `agents.usage_snapshot_json` 供现有列表、Dashboard 和限额状态兼容使用。 +当前管理端已先落地 Model Gateway 用量账本的最小闭环：Gateway 在模型请求完成后向管理端内部接口写入 `model_usage_events`，事件要求包含 `agentId` 和 `senderId`，并用 `eventId`/`requestId` 幂等去重。管理端提供 `GET /api/usage/summary` 和 `GET /api/usage/agents/{agentId}/senders` 查询接口，列表、Dashboard 和限额状态直接聚合账本；`agents.usage_snapshot_json` 仅作为旧数据库兼容字段保留，不再由用量同步任务刷新。 该 MVP 仍运行在现有管理服务的 SQLite 中，尚未完成独立 Model Gateway HTTP 代理、OpenClaw Provider base URL 切换、上游密钥移出 OpenClaw 和可信 Runtime/Client 身份桥接；这些仍属于 B1-B7 后续迁移范围，不能以当前用量接口推断为已完成。 diff --git a/packages/daedalus-agent-management-web/README.md b/packages/daedalus-agent-management-web/README.md index b8263d2..c31da24 100644 --- a/packages/daedalus-agent-management-web/README.md +++ b/packages/daedalus-agent-management-web/README.md @@ -12,9 +12,9 @@ - 业务系统管理、连通性测试、调用日志和统计 - 面向 CUI 与 `daedalus-agent-apps-invoke-cli serve` 的内部/外部 API；invoke CLI client 不直接访问这些 API - 本地 SQLite 缓存目录、状态、日志、统计和 Agent/用户 Token 用量展示；Model Gateway 保存按 `agentId`、`senderId` 记录的用量事实账本 -- 启动同步、定时同步和手动同步共用同一套同步计划 +- 启动同步、定时同步和手动同步共用同一套目录同步计划 -列表、状态和统计统一从 SQLite 读取。远端目录通过 `catalog.snapshot` 同步刷新；模型用量由 Model Gateway 在 `model_usage_events` 账本中按 `agentId`、`senderId` 记录，管理端刷新 `agents.usage_snapshot_json` 作为兼容展示快照；版本信息启动时通过 `version.list` 拉取并缓存到 `component_versions`。 +列表、状态和统计统一从 SQLite 读取。远端目录通过 `catalog.snapshot` 同步刷新；模型用量由 Model Gateway 在 `model_usage_events` 账本中按 `agentId`、`senderId` 记录，管理端直接聚合账本，不再执行用量同步或刷新 `agents.usage_snapshot_json`；版本信息启动时通过 `version.list` 拉取并缓存到 `component_versions`。 ## 目录结构 @@ -86,11 +86,9 @@ npm run test:full 1. 等待 `service.ping` 可用。 2. 执行 `version.list` 并写入 `component_versions`。 3. 执行 `catalog.snapshot`。 -4. 如果目录快照同步成功，60 秒后执行第一次 Model Gateway 用量快照刷新。 -5. 每次 `catalog.snapshot` 完成后 120 秒再次执行。 -6. 每次用量快照刷新完成后 120 秒再次执行。 +4. 每次 `catalog.snapshot` 完成后 120 秒再次执行。 -设置页“同步数据”按钮会先执行 `catalog.snapshot`，再从 Model Gateway 用量账本刷新快照。同步过程显示蒙层，完成后用 toast 提示结果。 +设置页“同步目录”按钮只执行 `catalog.snapshot`。用量由 Model Gateway 事件写入后实时聚合，页面不提供用量同步任务或状态表。 ## 对外 API @@ -103,7 +101,7 @@ npm run test:full - `GET /api/external/agents`（未传 `userName`/`id` 时仅返回“全部用户”可用的 Agent） - `GET /api/external/agents?userName=zhangsan`（全部用户模式不检查用户名是否存在；指定用户模式按显式授权筛选，也支持使用 `id` 作为身份参数） -- `POST /api/external/agents/:agentId/call`（请求体为 `senderId`、`text`，可选 `cookie`；若 `senderId` 不在非管理员 Agent 用户列表中，服务端会按每日限额 `-1`（不限额）幂等创建；服务端每次生成一次性 MQTT 关联会话，固定只返回 `kind=final` 最终结果，临时会话不写入数据库；若传入 `cookie`，仅在本次临时会话调用业务系统时使用，并在成功后刷新 Agent 用量计入每日配额） +- `POST /api/external/agents/:agentId/call`（请求体为 `senderId`、`text`，可选 `cookie`；若 `senderId` 不在非管理员 Agent 用户列表中，服务端会按每日限额 `-1`（不限额）幂等创建；服务端每次生成一次性 MQTT 关联会话，固定只返回 `kind=final` 最终结果，临时会话不写入数据库；若传入 `cookie`，仅在本次临时会话调用业务系统时使用；Model Gateway 写入的事件直接计入 Agent 每日配额） - `GET /api/external/session?userName=zhangsan&agentId=finance-agent`（Agent 为“全部用户”时不要求用户名已在员工列表中） - `GET /api/external/context?sessionId=...` @@ -124,7 +122,7 @@ API 文档： - `GET /api/internal/app-invoke/base-url?applicationName=...` - `POST /api/internal/app-invoke/logs` - `GET /api/internal/cui/agents` -- `POST /api/internal/model-gateway/usage-events`：Model Gateway 写入模型用量事件，必须携带 `agentId`、`senderId`；`eventId`/`requestId` 幂等。 +- `POST /api/internal/model-gateway/usage-events`：Model Gateway 写入模型用量事件，必须携带 `agentId`、`senderId`；`eventId`/`requestId` 幂等。Runtime Token 的 `agentId=runtime`、`senderId=unknown` 观测事件不计入用量统计，由 MQTT 插件补写带用户维度的计费事件。 管理员用量查询 API： diff --git a/packages/daedalus-agent-management-web/docs/spec.md b/packages/daedalus-agent-management-web/docs/spec.md index 94a77d3..43a573a 100644 --- a/packages/daedalus-agent-management-web/docs/spec.md +++ b/packages/daedalus-agent-management-web/docs/spec.md @@ -29,7 +29,7 @@ OpenClaw 侧真实目录通过同步任务写入本地数据库。页面读取 - 每日 Token 限额 - 状态：`normal`、`disabled`、`overlimit` -模型 Token 用量事实由 Model Gateway 写入 `model_usage_events`，每条事件带可信链路字段以及 `agentId`、`senderId`、实际 Provider/Model、输入/输出/缓存/总 Token 和计费标记。管理端从本地 Gateway 账本生成 `agents.usage_snapshot_json` 兼容展示快照，并据此刷新每日限额状态；快照同时保留当天 senderId 明细。超过每日限额后状态可变为 `overlimit`，并从 CUI 可用目录中排除。 +模型 Token 用量事实由 Model Gateway 写入 `model_usage_events`，每条事件带可信链路字段以及 `agentId`、`senderId`、实际 Provider/Model、输入/输出/缓存/总 Token 和计费标记。管理端的配额判断、Agent 列表和 Dashboard 均直接聚合本地 Gateway 账本；不再生成或刷新用量快照。超过每日限额后状态可变为 `overlimit`，并从 CUI 可用目录中排除。 ### Agent 平台用户 @@ -44,7 +44,7 @@ OpenClaw 侧真实目录通过同步任务写入本地数据库。页面读取 ### Agent 同步调用 -`POST /api/external/agents/{agentId}/call` 为指定 Agent 提供同步文本调用，JSON 请求体固定包含 `senderId` 和 `text`，可选包含调用方已有的 HZG `cookie`，不要求调用方传入 `sessionId`。如果 `senderId` 尚未出现在非管理员 Agent 用户列表 `agent_users` 中，控制台会以每日限额 `-1`（不限额）自动幂等创建该用户，不会写入管理员账号表 `users`。控制台为每次请求生成一次性 MQTT 关联会话 ID，并在进程内注册该临时会话的 Agent、sender 和 Cookie，仅在本次请求期间使用；结束后立即废弃，临时 session ID、Cookie 不写入 `external_sessions`、`external_session_cookies` 或调用日志的 `session_id`。控制台将请求转发到 Agent MQTT 入站 Topic，并设置 `disableBlockStreaming: true`；只等待匹配该临时 ID 的 `kind: "final"` 消息，忽略中间 `block` 消息后返回最终结果。Agent 通过 invoke service 调用业务系统时，使用该临时会话中的 Cookie；未传 Cookie 时按现有 HZG Cookie 获取流程在内存中临时生成。成功取得最终结果后，控制台刷新 Agent 用量快照，使本次调用产生的 Token 计入每日配额并更新超额状态。接口使用服务 Bearer Token 鉴权，本机请求遵循现有服务鉴权例外。 +`POST /api/external/agents/{agentId}/call` 为指定 Agent 提供同步文本调用，JSON 请求体固定包含 `senderId` 和 `text`，可选包含调用方已有的 HZG `cookie`，不要求调用方传入 `sessionId`。如果 `senderId` 尚未出现在非管理员 Agent 用户列表 `agent_users` 中，控制台会以每日限额 `-1`（不限额）自动幂等创建该用户，不会写入管理员账号表 `users`。控制台为每次请求生成一次性 MQTT 关联会话 ID，并在进程内注册该临时会话的 Agent、sender 和 Cookie，仅在本次请求期间使用；结束后立即废弃，临时 session ID、Cookie 不写入 `external_sessions`、`external_session_cookies` 或调用日志的 `session_id`。控制台将请求转发到 Agent MQTT 入站 Topic，并设置 `disableBlockStreaming: true`；只等待匹配该临时 ID 的 `kind: "final"` 消息，忽略中间 `block` 消息后返回最终结果。Agent 通过 invoke service 调用业务系统时，使用该临时会话中的 Cookie；未传 Cookie 时按现有 HZG Cookie 获取流程在内存中临时生成。成功取得最终结果后，Model Gateway 写入的用量事件会直接计入每日配额并更新超额状态。接口使用服务 Bearer Token 鉴权，本机请求遵循现有服务鉴权例外。 ### 模板 @@ -70,7 +70,7 @@ Provider 表示 OpenAI 兼容模型服务的 `baseUrl + SK`。Model 归属 Provi - 应用启动时执行一次 - 每次完成后 120 秒再次执行 -- 管理员点击“同步数据”时先执行 +- 管理员点击“同步目录”时执行 行为： @@ -81,24 +81,17 @@ Provider 表示 OpenAI 兼容模型服务的 `baseUrl + SK`。Model 归属 Provi `catalog.snapshot` 只携带同步落库所需字段，避免单条消息过大。 -### Model Gateway 用量同步 - -触发时机： - -- 启动 `catalog.snapshot` 成功后 60 秒执行第一次 -- 每次完成后 120 秒再次执行 -- 管理员点击“同步数据”时，在 `catalog.snapshot` 后执行 +### Model Gateway 用量账本 行为： -- Model Gateway 在每次模型请求完成后调用 `POST /api/internal/model-gateway/usage-events` 写入用量事件 +- Model Gateway 在每次模型请求完成后调用 `POST /api/internal/model-gateway/usage-events` 写入用量事件；Runtime Token 请求产生的 `agentId=runtime`、`senderId=unknown` 事件作为不可计费观测记录，由 OpenClaw MQTT 插件的 `llm_output` 事件提供带用户维度的计费记录，避免同一请求重复计量 - 用 `eventId` 和 `requestId` 做幂等去重，`billable=false` 的事件不进入统计 - 通过 `GET /api/usage/summary` 支持按 Agent、senderId、Agent+senderId、模型、天和小时查询 - 通过 `GET /api/usage/agents/{agentId}/senders` 查询指定 Agent 的 senderId 明细 -- 从 Gateway 账本生成所有本地 Agent 的日/周/月/总量快照 -- 写入 `agents.usage_snapshot_json`，快照标记 `source=model-gateway` 并包含 sender 明细 -- 刷新每日限额状态 -- 更新 `usage_refresh_state` +- 账本是用量的唯一事实源，按 Agent、senderId、Agent+senderId、模型、天和小时直接聚合 +- 配额检查和 Dashboard 查询使用账本实时结果，不依赖同步任务 +- `agents.usage_snapshot_json` 与 `usage_refresh_state` 仅作为旧数据库兼容字段保留；账本可用时不参与统计，也不会由同步任务刷新 ### 总览用量表 @@ -112,9 +105,9 @@ Provider 表示 OpenAI 兼容模型服务的 `baseUrl + SK`。Model 归属 Provi - 访问 Token 管理 - 服务操作 - 目录快照同步状态表 -- Model Gateway 用量同步状态表 +- Model Gateway 用量通过账本实时查询，不提供用量同步状态表 -服务操作包含“同步数据”、服务器状态、服务器诊断和重启服务。 +服务操作包含“同步目录”、服务器状态、服务器诊断和重启服务；同步目录只调用 `catalog.snapshot`，不刷新用量。 ## 技术方案 diff --git a/packages/daedalus-agent-management-web/server/management-src/capabilities/agent/lifecycle-service.ts b/packages/daedalus-agent-management-web/server/management-src/capabilities/agent/lifecycle-service.ts index b7ec889..8886567 100644 --- a/packages/daedalus-agent-management-web/server/management-src/capabilities/agent/lifecycle-service.ts +++ b/packages/daedalus-agent-management-web/server/management-src/capabilities/agent/lifecycle-service.ts @@ -24,44 +24,6 @@ import { import { ensureSharedMode } from "../shared-permissions.js"; import { atomicWriteFile } from "../../tools/atomic-file.js"; -const USAGE_QUERY_LIMIT = 100000; -const GATEWAY_CALL_TIMEOUT_MS = "30000"; - -function toIsoDate(value: Date): string { - const year = value.getFullYear(); - const month = String(value.getMonth() + 1).padStart(2, "0"); - const day = String(value.getDate()).padStart(2, "0"); - return `${year}-${month}-${day}`; -} - -function startOfDay(value: Date): Date { - const result = new Date(value); - result.setHours(0, 0, 0, 0); - return result; -} - -function extractTotalTokens(payload: unknown): number | null { - if (!payload || typeof payload !== "object") { - return null; - } - - if (Object.prototype.hasOwnProperty.call(payload, "totalTokens")) { - return Number((payload as Record).totalTokens ?? 0); - } - - const totals = ( - "totals" in payload && payload.totals && typeof payload.totals === "object" - ? payload.totals - : null - ) as Record | null; - - if (!totals || !Object.prototype.hasOwnProperty.call(totals, "totalTokens")) { - return null; - } - - return Number(totals.totalTokens ?? 0); -} - function firstText(...values: unknown[]): string { for (const value of values) { if (typeof value === "string" && value.trim()) { @@ -78,63 +40,10 @@ function addGatewayPrefix(value: string): string { return `daedalus-gateway/${normalized}`; } -function addUsageByAgent(target: Map, agentId: string, source: unknown): void { - const normalizedAgentId = agentId.trim(); - if (!normalizedAgentId) { - return; - } - - const totalTokens = extractTotalTokens(source); - if (totalTokens === null) { - return; - } - - target.set(normalizedAgentId, (target.get(normalizedAgentId) ?? 0) + totalTokens); -} - -function extractAllAgentUsage(payload: unknown): Map { - if (!payload || typeof payload !== "object") { - throw new Error("agent.usage.list returned an empty payload"); - } - - const source = payload as { - aggregates?: { - byAgent?: unknown[]; - }; - sessions?: unknown[]; - }; - const result = new Map(); - - if (Array.isArray(source.aggregates?.byAgent)) { - for (const item of source.aggregates.byAgent) { - if (!item || typeof item !== "object") { - continue; - } - const entry = item as Record; - addUsageByAgent(result, firstText(entry.agentId, entry.agent_id, entry.id), entry.totals ?? entry); - } - return result; - } - - if (Array.isArray(source.sessions)) { - for (const item of source.sessions) { - if (!item || typeof item !== "object") { - continue; - } - const entry = item as Record; - addUsageByAgent(result, firstText(entry.agentId, entry.agent_id), entry.usage ?? entry.totals ?? entry); - } - return result; - } - - throw new Error("agent.usage.list did not return per-agent usage fields"); -} - export class AgentLifecycleService { constructor( private readonly cli: CliRunner, - private readonly env: EnvironmentConfig, - private readonly now: () => Date = () => new Date() + private readonly env: EnvironmentConfig ) {} async listAgents(managedAgents: ManagedAgentRecord[]): Promise { @@ -307,66 +216,6 @@ export class AgentLifecycleService { }; } - async queryAllUsage(agentIds?: string[]): Promise { - const normalizedAgentIds = Array.isArray(agentIds) - ? [...new Set(agentIds.map((item) => String(item || "").trim()).filter(Boolean))] - : []; - const targetAgentIds = normalizedAgentIds.length > 0 - ? normalizedAgentIds - : (await this.listConfiguredAgents()) - .map((agent) => String(agent.id || "").trim()) - .filter(Boolean); - - if (targetAgentIds.length === 0) { - return { - generatedAt: toChinaISOString(), - items: [] - }; - } - - const today = startOfDay(this.now()); - const startDate = toIsoDate(today); - const endDate = toIsoDate(today); - - const remoteUsage = await this.queryAllUsageWindow(startDate, endDate); - const usageByAgent = extractAllAgentUsage(remoteUsage); - const cacheStatus = remoteUsage && typeof remoteUsage === "object" - ? (remoteUsage as { cacheStatus?: unknown }).cacheStatus - : undefined; - - return { - generatedAt: toChinaISOString(), - ...(cacheStatus ? { cacheStatus } : {}), - items: targetAgentIds.map((agentId) => ({ - agentId, - usage: { - daily: usageByAgent.get(agentId) ?? 0 - } - })) - }; - } - - private async queryAllUsageWindow(startDate: string, endDate: string): Promise { - const params = JSON.stringify({ - startDate, - endDate, - agentScope: "all", - limit: USAGE_QUERY_LIMIT, - includeContextWeight: false - }); - - return await this.safeJson([ - "gateway", - "call", - "--json", - "--timeout", - GATEWAY_CALL_TIMEOUT_MS, - "--params", - params, - "sessions.usage" - ]); - } - private async listConfiguredAgents(): Promise { + if (!result || typeof result !== "object") { + return; + } + + const status = String((result as { result?: unknown }).result || "").toLowerCase(); + if (status !== "scheduled" && status !== "coalesced" && status !== "deferred") { + return; + } + + const restart = (result as { restart?: { delayMs?: unknown } }).restart; + const delayMs = typeof restart?.delayMs === "number" && Number.isFinite(restart.delayMs) + ? Math.max(0, Math.floor(restart.delayMs)) + : 0; + const settleMs = Math.max(0, AgentOperationsService.GATEWAY_RESTART_SETTLE_MS); + const waitMs = delayMs + settleMs; + if (waitMs === 0) { + return; + } + + await new Promise((resolve) => setTimeout(resolve, waitMs)); + } + private async waitForGatewayReady(): Promise { const deadline = Date.now() + AgentOperationsService.GATEWAY_READY_TIMEOUT_MS; diff --git a/packages/daedalus-agent-management-web/server/management-src/capabilities/agent/service.ts b/packages/daedalus-agent-management-web/server/management-src/capabilities/agent/service.ts index 6ab87af..599478f 100644 --- a/packages/daedalus-agent-management-web/server/management-src/capabilities/agent/service.ts +++ b/packages/daedalus-agent-management-web/server/management-src/capabilities/agent/service.ts @@ -201,10 +201,6 @@ export class OpenClawManager { } } - async queryAllUsage(params: { agentIds?: string[] } = {}): Promise { - return await this.lifecycleService.queryAllUsage(params.agentIds); - } - async setAgentModel(params: AgentModelSetParams): Promise { const result = await this.lifecycleService.setAgentModel(params); await this.operationsService.maybeRefreshGatewayAfterManagedTopologyChange(params.restart, `agent.model.set:${params.agentId}`); diff --git a/packages/daedalus-agent-management-web/server/management-src/capabilities/agent/usage-list.ts b/packages/daedalus-agent-management-web/server/management-src/capabilities/agent/usage-list.ts deleted file mode 100644 index 616dfcc..0000000 --- a/packages/daedalus-agent-management-web/server/management-src/capabilities/agent/usage-list.ts +++ /dev/null @@ -1,5 +0,0 @@ -import type { CapabilityHandler } from "../context.js"; - -export const agentUsageList: CapabilityHandler = async (request, context) => { - return await context.agentManager.queryAllUsage((request.params ?? {}) as { agentIds?: string[] }); -}; diff --git a/packages/daedalus-agent-management-web/server/management-src/capabilities/context.ts b/packages/daedalus-agent-management-web/server/management-src/capabilities/context.ts index 43c13ca..cb9a11d 100644 --- a/packages/daedalus-agent-management-web/server/management-src/capabilities/context.ts +++ b/packages/daedalus-agent-management-web/server/management-src/capabilities/context.ts @@ -53,7 +53,6 @@ export interface AgentCapacityService { enableAgent(params: AgentMutationParams): Promise; disableAgent(params: AgentMutationParams): Promise; deleteAgent(params: AgentMutationParams): Promise<{ agentId: string; accountId: string }>; - queryAllUsage(params?: { agentIds?: string[] }): Promise; setAgentModel(params: AgentModelSetParams): Promise; setAgentName(params: AgentNameSetParams): Promise; updateAgentDocs(params: AgentDocsUpdateParams): Promise<{ path: string }>; diff --git a/packages/daedalus-agent-management-web/server/management-src/capabilities/registry.ts b/packages/daedalus-agent-management-web/server/management-src/capabilities/registry.ts index afdcf22..d92a6f1 100644 --- a/packages/daedalus-agent-management-web/server/management-src/capabilities/registry.ts +++ b/packages/daedalus-agent-management-web/server/management-src/capabilities/registry.ts @@ -7,7 +7,6 @@ import { agentEnable } from "./agent/enable.js"; import { agentGet } from "./agent/get.js"; import { agentModelSet } from "./agent/model-set.js"; import { agentNameSet } from "./agent/name-set.js"; -import { agentUsageList } from "./agent/usage-list.js"; import { agentMarkdownDelete } from "./agent/markdown-delete.js"; import { agentMarkdownList } from "./agent/markdown-list.js"; import { agentMarkdownRead } from "./agent/markdown-read.js"; @@ -56,7 +55,6 @@ export const capabilityHandlers: Record = { "agent.enable": agentEnable, "agent.disable": agentDisable, "agent.delete": agentDelete, - "agent.usage.list": agentUsageList, "agent.model.set": agentModelSet, "agent.name.set": agentNameSet, "agent.docs.update": agentDocsUpdate, diff --git a/packages/daedalus-agent-management-web/server/management-src/service.ts b/packages/daedalus-agent-management-web/server/management-src/service.ts index 6221e1f..483e60e 100644 --- a/packages/daedalus-agent-management-web/server/management-src/service.ts +++ b/packages/daedalus-agent-management-web/server/management-src/service.ts @@ -133,7 +133,6 @@ const QUEUE_BYPASS_ACTIONS = new Set([ "service.ping", "version.list", "catalog.snapshot", - "agent.usage.list", "skills.global.list", "gateway.status" ]); diff --git a/packages/daedalus-agent-management-web/server/management-test/capabilities-registry.test.ts b/packages/daedalus-agent-management-web/server/management-test/capabilities-registry.test.ts index 346a5c8..46e6a1f 100644 --- a/packages/daedalus-agent-management-web/server/management-test/capabilities-registry.test.ts +++ b/packages/daedalus-agent-management-web/server/management-test/capabilities-registry.test.ts @@ -16,7 +16,6 @@ const expectedActions = [ "agent.enable", "agent.disable", "agent.delete", - "agent.usage.list", "agent.model.set", "agent.name.set", "agent.docs.update", @@ -81,7 +80,6 @@ function createContext(calls: Array<{ method: string; args: unknown[] }>): Capab enableAgent: respond("agentManager.enableAgent") as never, disableAgent: respond("agentManager.disableAgent") as never, deleteAgent: respond("agentManager.deleteAgent") as never, - queryAllUsage: respond("agentManager.queryAllUsage"), setAgentModel: respond("agentManager.setAgentModel"), setAgentName: respond("agentManager.setAgentName"), updateAgentDocs: respond("agentManager.updateAgentDocs") as never, @@ -218,12 +216,6 @@ test("capability handlers delegate to the expected service methods", async () => expectedMethod: "agentManager.deleteAgent", expectedArgs: [{ agentId: "lowcode", restart: true }] }, - { - action: "agent.usage.list", - params: { agentIds: ["lowcode", "support"] }, - expectedMethod: "agentManager.queryAllUsage", - expectedArgs: [{ agentIds: ["lowcode", "support"] }] - }, { action: "agent.model.set", params: { agentId: "lowcode", model: { primary: "openai/gpt-5.5" } }, diff --git a/packages/daedalus-agent-management-web/server/management-test/coverage-contract.test.ts b/packages/daedalus-agent-management-web/server/management-test/coverage-contract.test.ts index 7f1ac18..598f41f 100644 --- a/packages/daedalus-agent-management-web/server/management-test/coverage-contract.test.ts +++ b/packages/daedalus-agent-management-web/server/management-test/coverage-contract.test.ts @@ -16,7 +16,6 @@ import { validateAgentId, validateTemplateName } from "../management-src/capabilities/agent/support.js"; -import { agentUsageList } from "../management-src/capabilities/agent/usage-list.js"; import { extractZipToDirectory } from "../management-src/tools/zip-archive.js"; import { readLocalArtifact } from "../management-src/tools/local-artifact.js"; import { versionList } from "../management-src/capabilities/version/list.js"; @@ -135,22 +134,9 @@ test("reads and filters built-in component manifests", async () => { } }); -test("defaults capability params and forwards agent usage queries", async () => { +test("defaults capability params", () => { assert.deepEqual(readCapabilityParams({ params: { key: "value" } } as any), { key: "value" }); assert.deepEqual(readCapabilityParams({} as any), {}); - - const calls: unknown[] = []; - const context = { - agentManager: { - async queryAllUsage(params: unknown) { - calls.push(params); - return { ok: true }; - } - } - } as any; - assert.deepEqual(await agentUsageList({ params: { agentIds: ["alpha"] } } as any, context), { ok: true }); - assert.deepEqual(await agentUsageList({} as any, context), { ok: true }); - assert.deepEqual(calls, [{ agentIds: ["alpha"] }, {}]); }); test("maps catalog snapshots across empty, legacy, and model variants", async () => { diff --git a/packages/daedalus-agent-management-web/server/management-test/lifecycle-service-coverage.test.ts b/packages/daedalus-agent-management-web/server/management-test/lifecycle-service-coverage.test.ts index aef9e43..3d67d7e 100644 --- a/packages/daedalus-agent-management-web/server/management-test/lifecycle-service-coverage.test.ts +++ b/packages/daedalus-agent-management-web/server/management-test/lifecycle-service-coverage.test.ts @@ -105,93 +105,6 @@ test("lifecycle agent listing covers malformed ids, config defaults, and disable } }); -test("lifecycle usage accepts aggregate and session payload variants", async () => { - const root = await mkdtemp(path.join(tmpdir(), "lifecycle-usage-coverage-")); - const env = createEnv(root); - const cli = new QueueCliRunner([ - { - aggregates: { - byAgent: [ - null, - { agentId: " ", totals: { totalTokens: 10 } }, - { id: "alpha", totalTokens: null }, - { agent_id: "alpha", totals: { totalTokens: 2 } }, - { agentId: "missing", totals: null }, - { agentId: "missing", totals: {} } - ] - } - }, - { - sessions: [ - null, - { agentId: " ", usage: { totalTokens: 1 } }, - { agent_id: "alpha", usage: null, totals: { totalTokens: 2 } }, - { agentId: "alpha", usage: 7 }, - { agentId: "beta", usage: { totals: {} } }, - { agentId: "beta", usage: { totals: 1 } }, - { agentId: "beta", usage: { totals: { totalTokens: null } } }, - { agentId: "gamma", totalTokens: 3 } - ], - cacheStatus: 0 - } - ]); - const service = new AgentLifecycleService(cli as never, env, () => new Date("2026-08-17T09:30:00.000Z")); - - try { - const aggregateResult = await service.queryAllUsage([ - " alpha ", - "", - "alpha", - "missing", - "zero" - ]) as { items: Array<{ agentId: string; usage: { daily: number } }>; cacheStatus?: unknown }; - assert.deepEqual(aggregateResult.items, [ - { agentId: "alpha", usage: { daily: 2 } }, - { agentId: "missing", usage: { daily: 0 } }, - { agentId: "zero", usage: { daily: 0 } } - ]); - assert.equal("cacheStatus" in aggregateResult, false); - - const sessionResult = await service.queryAllUsage(["alpha", "beta", "gamma", "missing"]) as { - items: Array<{ agentId: string; usage: { daily: number } }>; - cacheStatus?: unknown; - }; - assert.deepEqual(sessionResult.items, [ - { agentId: "alpha", usage: { daily: 2 } }, - { agentId: "beta", usage: { daily: 0 } }, - { agentId: "gamma", usage: { daily: 3 } }, - { agentId: "missing", usage: { daily: 0 } } - ]); - assert.equal("cacheStatus" in sessionResult, false); - assert.equal(cli.runJsonCalls.length, 2); - } finally { - await rm(root, { recursive: true, force: true }); - } -}); - -test("lifecycle usage handles empty configuration and rejects invalid payloads", async () => { - const root = await mkdtemp(path.join(tmpdir(), "lifecycle-usage-errors-")); - const env = createEnv(root); - - try { - await writeConfig(env.openclawConfigPath, { agents: { list: [{ id: 0 }] } }); - const emptyIdService = new AgentLifecycleService(new QueueCliRunner() as never, env); - const emptyIdResult = await emptyIdService.queryAllUsage([]) as { generatedAt: string; items: unknown[] }; - assert.equal(typeof emptyIdResult.generatedAt, "string"); - assert.deepEqual(emptyIdResult.items, []); - - await writeConfig(env.openclawConfigPath, { agents: { list: {} } }); - const emptyListService = new AgentLifecycleService(new QueueCliRunner() as never, env); - const emptyList = await emptyListService.queryAllUsage(); - assert.deepEqual((emptyList as { items: unknown[] }).items, []); - - const invalidService = new AgentLifecycleService(new QueueCliRunner([null, {}]) as never, env); - await assert.rejects(invalidService.queryAllUsage(["alpha"]), /empty payload/); - await assert.rejects(invalidService.queryAllUsage(["alpha"]), /per-agent usage fields/); - } finally { - await rm(root, { recursive: true, force: true }); - } -}); test("lifecycle config mutations cover missing lists and validation errors", async () => { const root = await mkdtemp(path.join(tmpdir(), "lifecycle-mutation-coverage-")); diff --git a/packages/daedalus-agent-management-web/server/management-test/openclaw-manager.test.ts b/packages/daedalus-agent-management-web/server/management-test/openclaw-manager.test.ts index 92e08e7..de245e6 100644 --- a/packages/daedalus-agent-management-web/server/management-test/openclaw-manager.test.ts +++ b/packages/daedalus-agent-management-web/server/management-test/openclaw-manager.test.ts @@ -459,6 +459,41 @@ test("restartGateway uses openclaw restart", async () => { assert.deepEqual(cli.runJsonCalls, [GATEWAY_READY_CALL]); }); +test("restartGateway waits for a scheduled restart before probing readiness", async () => { + const originalSettleMs = (AgentOperationsService as any).GATEWAY_RESTART_SETTLE_MS; + (AgentOperationsService as any).GATEWAY_RESTART_SETTLE_MS = 0; + const restartDelayMs = 2500; + const startedAt = Date.now(); + const probeDeltas: number[] = []; + const cli = { + async run(args: string[]): Promise { + assert.deepEqual(args, ["gateway", "restart", "--safe", "--skip-deferral", "--json"]); + return { + stdout: JSON.stringify({ + ok: true, + result: "scheduled", + restart: { delayMs: restartDelayMs } + }), + stderr: "", + exitCode: 0 + }; + }, + async runJson(args: string[]): Promise { + assert.deepEqual(args, GATEWAY_READY_CALL); + probeDeltas.push(Date.now() - startedAt); + return { rpc: { ok: true } }; + } + }; + const manager = new AgentOperationsService(cli as never, {} as never, createEnv()); + + try { + await manager.restartGateway({ safe: true, skipDeferral: true }); + assert(probeDeltas[0] >= restartDelayMs, `readiness was probed before the scheduled restart: ${probeDeltas}`); + } finally { + (AgentOperationsService as any).GATEWAY_RESTART_SETTLE_MS = originalSettleMs; + } +}); + test("restartGateway accepts successful restart without JSON output", async () => { const cli = new MockCliRunner([{ rpc: { ok: true } }], { stdout: " ", stderr: "", exitCode: 0 }); const doctor = new MockCliRunner(); @@ -1423,76 +1458,6 @@ test("setAgentModel restarts gateway by default", async () => { }); }); -test("queryAllUsage returns every configured agent usage in one capability response", async () => { - const previousEnv = { ...process.env }; - const root = await mkdtemp(path.join(tmpdir(), "oc-mgmt-agent-usage-list-")); - const configPath = path.join(root, ".openclaw", "openclaw.json"); - const stateStore = new ManagedStateStore(path.join(root, "managed-state.json")); - await writeOpenClawConfig(configPath, { - agents: { - list: [ - { id: "alpha", name: "Alpha" }, - { id: "beta", name: "Beta" } - ] - } - }); - const cli = new MockCliRunner([ - { - totals: { totalTokens: 8 }, - aggregates: { - byAgent: [ - { agentId: "beta", totals: { totalTokens: 5 } }, - { agentId: "alpha", totals: { totalTokens: 3 } } - ] - }, - cacheStatus: { status: "refreshing" } - } - ]); - const doctor = new MockCliRunner(); - const now = new Date("2026-05-27T10:00:00.000Z"); - const manager = new OpenClawManager( - cli as never, - doctor as never, - createEnv({ dataRoot: root, openclawConfigPath: configPath }), - stateStore - ); - const env = createEnv({ dataRoot: root, openclawConfigPath: configPath }); - (manager as any).lifecycleService = new (Object.getPrototypeOf((manager as any).lifecycleService).constructor)( - cli as never, - env, - () => new Date(now) - ); - - try { - process.env.OPENCLAW_HOME = "/root"; - process.env.OPENCLAW_STATE_DIR = "/root/.openclaw"; - process.env.OPENCLAW_CONFIG_PATH = "/root/.openclaw/openclaw.json"; - - const usage = await manager.queryAllUsage() as { items: Array<{ agentId: string; usage: { daily: number } }> }; - - assert.deepEqual(usage.items, [ - { agentId: "alpha", usage: { daily: 3 } }, - { agentId: "beta", usage: { daily: 5 } } - ]); - assert.deepEqual(cli.runJsonCalls, [ - [ - "gateway", - "call", - "--json", - "--timeout", - "30000", - "--params", - "{\"startDate\":\"2026-05-27\",\"endDate\":\"2026-05-27\",\"agentScope\":\"all\",\"limit\":100000,\"includeContextWeight\":false}", - "sessions.usage" - ] - ]); - assert.equal(cli.runJsonEnvs[0]?.OPENCLAW_STATE_DIR, env.openclawStateDir); - assert.equal(cli.runJsonEnvs[0]?.OPENCLAW_CONFIG_PATH, configPath); - assert.equal(cli.runJsonEnvs[0]?.OPENCLAW_GATEWAY_TOKEN, env.gatewayAuthToken); - } finally { - process.env = previousEnv; - } -}); test("enableAgent and disableAgent restart the gateway by default in host mode", async () => { const root = await mkdtemp(path.join(tmpdir(), "oc-mgmt-agent-toggle-host-")); diff --git a/packages/daedalus-agent-management-web/server/management-test/service-queue.test.ts b/packages/daedalus-agent-management-web/server/management-test/service-queue.test.ts index b74f105..8c958ef 100644 --- a/packages/daedalus-agent-management-web/server/management-test/service-queue.test.ts +++ b/packages/daedalus-agent-management-web/server/management-test/service-queue.test.ts @@ -6,7 +6,6 @@ test("bypasses the serial management queue for lightweight status requests", () assert.equal(shouldBypassManagementQueue("service.ping"), true); assert.equal(shouldBypassManagementQueue("version.list"), true); assert.equal(shouldBypassManagementQueue("catalog.snapshot"), true); - assert.equal(shouldBypassManagementQueue("agent.usage.list"), true); assert.equal(shouldBypassManagementQueue("skills.global.list"), true); assert.equal(shouldBypassManagementQueue("gateway.status"), true); }); @@ -14,5 +13,6 @@ test("bypasses the serial management queue for lightweight status requests", () test("keeps mutating or potentially long management requests serialized", () => { assert.equal(shouldBypassManagementQueue("gateway.restart"), false); assert.equal(shouldBypassManagementQueue("agent.create"), false); + assert.equal(shouldBypassManagementQueue("agent.unknown"), false); assert.equal(shouldBypassManagementQueue("skills.global.install.local"), false); }); diff --git a/packages/daedalus-agent-management-web/server/src/api/routes/index.js b/packages/daedalus-agent-management-web/server/src/api/routes/index.js index f50126b..c26c1e7 100644 --- a/packages/daedalus-agent-management-web/server/src/api/routes/index.js +++ b/packages/daedalus-agent-management-web/server/src/api/routes/index.js @@ -49,27 +49,6 @@ function normalizeTimeoutMs(value, fallback = 120000, max = 120000) { return Math.max(1000, Math.min(max, Math.floor(parsed))); } -async function refreshUsageAfterExternalAgentCall(services, agentId) { - const refreshAgentUsage = services.catalogSyncService?.refreshAgentUsage; - if (typeof refreshAgentUsage !== "function") { - return; - } - - try { - const result = await refreshAgentUsage.call(services.catalogSyncService, { - trigger: "external-agent-call" - }); - if (result?.status === "failed") { - logger.warn({ - agentId, - error: result.error_message || "unknown usage refresh error" - }, "External agent call usage refresh failed"); - } - } catch (error) { - logger.warn({ agentId, error: errorDetails(error) }, "External agent call usage refresh failed"); - } -} - function normalizeSystemLogType(value) { const type = String(value || "").trim(); if (!SYSTEM_LOG_TYPE_VALUES.has(type)) { @@ -150,8 +129,7 @@ function getCatalogSyncStatuses(services) { llm_sync: services.catalogSyncService.getLlmSyncStatus(), skill_sync: services.catalogSyncService.getSkillSyncStatus(), template_sync: services.catalogSyncService.getTemplateSyncStatus(), - system_sync: services.catalogSyncService.getSystemSyncStatus(), - usage_refresh: services.catalogSyncService.getUsageRefreshStatus() + system_sync: services.catalogSyncService.getSystemSyncStatus() }; } @@ -356,6 +334,12 @@ export function createRoutes(services) { } catch (error) { throw badRequest(error instanceof Error ? error.message : "用量事件参数不合法"); } + if (result.inserted) { + services.updateNotifier?.broadcast?.("usage.updated", { + trigger: "model-gateway", + event_id: result.eventId + }); + } res.status(result.inserted ? 201 : 200).json(result); })); @@ -406,7 +390,6 @@ export function createRoutes(services) { text: payload.text, cookie: payload.cookie }); - await refreshUsageAfterExternalAgentCall(services, req.params.agentId); res.json(result); })); @@ -449,11 +432,19 @@ export function createRoutes(services) { })); router.get("/bootstrap", asyncRoute(async (req, res) => { + let versionInfo; + if (typeof services.portalService?.refreshVersionInfoCache === "function") { + versionInfo = await services.portalService.refreshVersionInfoCache({ + trigger: "bootstrap", + timeoutMs: normalizeTimeoutMs(services.env?.systemStatusTimeoutMs, 5000) + }); + } + res.json({ current_user: req.currentUser, settings: services.portalService.getSettings(), access_tokens: services.portalService.listAccessTokens(), - version_info: services.portalService.getVersionInfo(), + version_info: versionInfo || services.portalService.getVersionInfo(), ...getCatalogSyncStatuses(services), env_config: getRuntimeEnvConfig(services) }); @@ -1444,13 +1435,7 @@ export function createRoutes(services) { status: result?.catalog?.status, finished_at: result?.catalog?.finished_at }); - services.updateNotifier?.broadcast?.("usage.updated", { - trigger: "manual", - status: result?.usage?.status, - finished_at: result?.usage?.finished_at, - summary: result?.usage?.summary - }); - writeAuditLog(services, req, "同步数据", "手动触发 catalog.snapshot 与 Model Gateway 用量快照同步"); + writeAuditLog(services, req, "同步目录", "手动触发 catalog.snapshot，同步目录数据"); res.json({ result, ...getCatalogSyncStatuses(services) diff --git a/packages/daedalus-agent-management-web/server/src/app.js b/packages/daedalus-agent-management-web/server/src/app.js index b38d62a..97315d5 100644 --- a/packages/daedalus-agent-management-web/server/src/app.js +++ b/packages/daedalus-agent-management-web/server/src/app.js @@ -74,13 +74,13 @@ export function createServerApp() { const runtimeComponentHealthService = new RuntimeComponentHealthService({ env }); const containerService = new ContainerService({ env }); const modelGatewayUsageService = new ModelGatewayUsageService({ db }); - const catalogSyncService = new CatalogSyncService({ db, managementClient, auditLogService, env, usageService: modelGatewayUsageService }); + const catalogSyncService = new CatalogSyncService({ db, managementClient, auditLogService, env }); const portalService = new PortalService({ db, localArtifactStore, managementClient, authService }); const templateService = new TemplateService({ db, portalService, env }); portalService.cacheLocalVersionInfo(); const largeLanguageModelService = new LargeLanguageModelService({ db, managementClient }); const mcpService = new McpService({ db, managementClient, env }); - const agentService = new AgentService({ db, managementClient, localArtifactStore, env, mcpService }); + const agentService = new AgentService({ db, managementClient, localArtifactStore, env, mcpService, usageService: modelGatewayUsageService }); const systemService = new SystemService({ db, localArtifactStore, managementClient, runtimeEnvConfigService, env }); const transientSessionStore = new TransientExternalSessionStore(); const externalService = new ExternalService({ diff --git a/packages/daedalus-agent-management-web/server/src/background/index.js b/packages/daedalus-agent-management-web/server/src/background/index.js index 94d65eb..b911c15 100644 --- a/packages/daedalus-agent-management-web/server/src/background/index.js +++ b/packages/daedalus-agent-management-web/server/src/background/index.js @@ -83,10 +83,8 @@ export async function waitForManagementActionReady({ export function startBackgroundServices({ env, services }) { let catalogTimer = null; - let usageTimer = null; let systemStatusTimer = null; const scheduleDelayMs = 2 * 60 * 1000; - const usageStartupDelayMs = 60 * 1000; const systemStatusInitialDelayMs = normalizePositiveNumber( env?.systemStatusInitialDelayMs, DEFAULT_SYSTEM_STATUS_INITIAL_DELAY_MS @@ -125,26 +123,6 @@ export function startBackgroundServices({ env, services }) { }, delayMs); } - function scheduleUsageSync(delayMs) { - clearTimer(usageTimer); - usageTimer = setTimeout(async () => { - usageTimer = null; - try { - const result = await services.catalogSyncService.refreshAgentUsage({ trigger: "scheduled" }); - notifyUpdate(services, "usage.updated", { - trigger: "scheduled", - status: result?.status, - finished_at: result?.finished_at, - summary: result?.summary - }); - } catch (error) { - logger.error({ error: errorDetails(error) }, "Scheduled agent usage sync failed"); - } finally { - scheduleUsageSync(scheduleDelayMs); - } - }, delayMs); - } - async function refreshSystemStatus(trigger) { const tasks = []; if (typeof services.portalService?.refreshVersionInfoCache === "function") { @@ -244,9 +222,6 @@ export function startBackgroundServices({ env, services }) { finished_at: result?.finished_at }); logger.info({ trigger: "startup", status: result?.status }, "Startup catalog sync finished"); - if (result?.status === "success") { - scheduleUsageSync(usageStartupDelayMs); - } } catch (error) { logger.error({ error: errorDetails(error) }, "Startup catalog sync failed"); } finally { @@ -262,8 +237,6 @@ export function startBackgroundServices({ env, services }) { async stop() { clearTimer(catalogTimer); catalogTimer = null; - clearTimer(usageTimer); - usageTimer = null; clearTimer(systemStatusTimer); systemStatusTimer = null; diff --git a/packages/daedalus-agent-management-web/server/src/docs/openapi.js b/packages/daedalus-agent-management-web/server/src/docs/openapi.js index 60dad93..d9f3041 100644 --- a/packages/daedalus-agent-management-web/server/src/docs/openapi.js +++ b/packages/daedalus-agent-management-web/server/src/docs/openapi.js @@ -165,7 +165,7 @@ const networkAllowlistBody = jsonBody("运行时网络配置。", { agent_internal_network_allowlist: textField("Agent 可访问的内网 IP/CIDR 列表；可用逗号、空格或分号分隔。"), huozige_server_ip: textField("活字格服务器 IP 或域名；不能包含路径、查询参数或片段。") }); -const modelGatewayUsageEventBody = jsonBody("Model Gateway 用量事件。事件由网关在一次模型请求完成后写入，eventId/requestId 用于幂等去重。", { +const modelGatewayUsageEventBody = jsonBody("Model Gateway 用量事件。事件由网关在一次模型请求完成后写入，eventId/requestId 用于幂等去重；Runtime Token 的 agentId=runtime、senderId=unknown 观测事件不计入统计。", { eventId: textField("网关事件唯一 ID；重复事件不会重复计量。", { minLength: 1 }), requestId: textField("模型请求唯一 ID；重试请求使用同一 ID。"), traceId: textField("链路 trace ID，可选。"), @@ -182,7 +182,7 @@ const modelGatewayUsageEventBody = jsonBody("Model Gateway 用量事件。事件 cachedTokens: integerField("缓存命中 Token 数，必须为非负整数。", { minimum: 0 }), totalTokens: integerField("总 Token 数；不传时按输入加输出计算。", { minimum: 0 }), usageSource: textField("用量来源，例如 provider。"), - billable: booleanField("是否计入统计；默认 true。"), + billable: booleanField("是否计入统计；默认 true；Runtime Token 无用户观测事件会按不可计费处理。"), success: booleanField("模型请求是否成功；默认 true。"), errorCode: textField("失败错误码，可选。"), latencyMs: integerField("模型请求耗时，单位毫秒。", { minimum: 0 }), @@ -694,7 +694,7 @@ const ROUTES = [ route("get", "/settings/access-tokens", "系统设置", "查询管理 Access Token", "管理员查询已创建的服务访问令牌摘要。"), route("post", "/settings/access-tokens", "系统设置", "创建管理 Access Token", "创建随机服务 Bearer Token；完整 Token 仅在创建响应中返回。", { successStatus: 201 }), route("delete", "/settings/access-tokens/{token}", "系统设置", "删除管理 Access Token", "删除指定 Token，删除后立即失效。"), - route("post", "/settings/catalog-sync", "系统设置", "同步目录和用量", "管理员先同步 catalog.snapshot，再从 Model Gateway 用量账本刷新 Agent 快照；返回两个任务状态和摘要。"), + route("post", "/settings/catalog-sync", "系统设置", "同步目录", "管理员触发 catalog.snapshot，同步目录状态和摘要；用量由 Model Gateway 账本实时聚合，不执行用量刷新。"), route("post", "/settings/server-status", "系统设置", "查询服务状态", "调用 gateway.status 获取运行时服务状态；默认记录审计，可用 audit=false 关闭本次审计。", { query: [ queryParam("timeout_ms", "超时时间，默认 120000 毫秒。", { type: "integer", minimum: 1 }), diff --git a/packages/daedalus-agent-management-web/server/src/services/agent-quota.js b/packages/daedalus-agent-management-web/server/src/services/agent-quota.js index 4214398..5e10cb3 100644 --- a/packages/daedalus-agent-management-web/server/src/services/agent-quota.js +++ b/packages/daedalus-agent-management-web/server/src/services/agent-quota.js @@ -1,6 +1,7 @@ import { chinaDateKey } from "../utils/china-time.js"; export const AGENT_STATUSES = ["normal", "disabled", "overlimit"]; +const BILLABLE_USAGE_FILTER = "billable = 1 AND NOT (agent_id = 'runtime' AND sender_id = 'unknown')"; const QUOTA_SPECS = [ { key: "daily", limitKey: "daily_limit", label: "daily" } @@ -65,7 +66,7 @@ export function queryTodayTokenUsage(db, { agentId, senderId, now = new Date() } const dateKey = chinaDateKey(now); const startAt = `${dateKey}T00:00:00.000+08:00`; const endAt = `${addChinaDays(dateKey, 1)}T00:00:00.000+08:00`; - const filters = ["created_at >= ?", "created_at < ?", "billable = 1"]; + const filters = ["created_at >= ?", "created_at < ?", BILLABLE_USAGE_FILTER]; const params = [startAt, endAt]; if (agentId !== undefined && String(agentId).trim()) { filters.push("agent_id = ?"); diff --git a/packages/daedalus-agent-management-web/server/src/services/agent-service.js b/packages/daedalus-agent-management-web/server/src/services/agent-service.js index 7ee0522..ba81cce 100644 --- a/packages/daedalus-agent-management-web/server/src/services/agent-service.js +++ b/packages/daedalus-agent-management-web/server/src/services/agent-service.js @@ -242,6 +242,16 @@ function effectiveAgentStatus(row, db = null) { return db ? statusFromCurrentUsage(db, row, usage) : computeHealth(row, usage); } +function normalizeLedgerUsage(value) { + const source = value && typeof value === "object" ? (value.usage || value.tokens || value) : {}; + return { + daily: Number(source.daily ?? source.day ?? source.dailyTokens ?? source.today ?? 0), + weekly: Number(source.weekly ?? source.weeklyTokens ?? 0), + monthly: Number(source.monthly ?? source.monthlyTokens ?? 0), + total: Number(source.total ?? source.totalTokens ?? 0) + }; +} + function agentDirectoryItem(row, users, isPublic, db = null) { const remote = normalizeRemoteState(jsonParse(row.remote_state_json, {})); const item = { @@ -301,12 +311,13 @@ function runInTransaction(database, fn) { } export class AgentService { - constructor({ db, managementClient, localArtifactStore, env, mcpService }) { + constructor({ db, managementClient, localArtifactStore, env, mcpService, usageService = null }) { this.db = db; this.managementClient = managementClient; this.localArtifactStore = localArtifactStore; this.env = env; this.mcpService = mcpService; + this.usageService = usageService; } localRows() { @@ -747,8 +758,14 @@ export class AgentService { async listAgents() { const rows = this.localRows(); + let usageByAgent = new Map(); + if (this.usageService && typeof this.usageService.listAgentUsage === "function") { + const result = this.usageService.listAgentUsage({ agentIds: rows.map((row) => row.slug) }); + usageByAgent = new Map((result?.items || []).map((item) => [item.agentId, normalizeLedgerUsage(item.usage)])); + } return rows.map((row) => { - const usage = normalizeUsage(jsonParse(row.usage_snapshot_json, {}).usage); + const usage = usageByAgent.get(row.slug) + || (this.usageService ? { daily: 0, weekly: 0, monthly: 0, total: 0 } : normalizeUsage(jsonParse(row.usage_snapshot_json, {}).usage)); const remote = normalizeRemoteState(jsonParse(row.remote_state_json, {})); const permissions = this.getPermissions(row.id); return { @@ -1549,10 +1566,9 @@ export class AgentService { daily_limit: Number(payload.daily_limit ?? current.daily_limit) }; if (nextStatus === "normal") { - const usage = normalizeUsage(jsonParse(current.usage_snapshot_json, {}).usage); const violation = hasUsageLedger(this.db) ? findTodayQuotaViolation(this.db, nextQuotaFields, { agentId: current.slug }) - : findQuotaViolation(nextQuotaFields, usage); + : findQuotaViolation(nextQuotaFields, normalizeUsage(jsonParse(current.usage_snapshot_json, {}).usage)); if (violation) { throw badRequest(formatQuotaViolationMessage(current, violation), { agentId: current.slug, diff --git a/packages/daedalus-agent-management-web/server/src/services/catalog-sync-service.js b/packages/daedalus-agent-management-web/server/src/services/catalog-sync-service.js index 93a23df..d19ad48 100644 --- a/packages/daedalus-agent-management-web/server/src/services/catalog-sync-service.js +++ b/packages/daedalus-agent-management-web/server/src/services/catalog-sync-service.js @@ -1,18 +1,9 @@ import { jsonParse, withSerializedTransaction } from "../db/index.js"; import { toChinaISOString } from "../utils/china-time.js"; import { - normalizeAgentStatus, - normalizeUsage, - statusAfterUsageRefresh + normalizeAgentStatus } from "./agent-quota.js"; -const USAGE_FIELDS = [ - "daily", - "day", - "dailyTokens", - "today", - "totalTokens" -]; const DEFAULT_TEMPLATE_NAME = "default"; const STARTUP_SYNC_STATE_TABLES = [ "agent_sync_state", @@ -41,41 +32,6 @@ const CATALOG_SYNC_STATES = [ ]; const CATALOG_SNAPSHOT_VERSION = 1; -function hasUsageFields(payload) { - if (!payload || typeof payload !== "object") { - return false; - } - - return USAGE_FIELDS.some((field) => Object.prototype.hasOwnProperty.call(payload, field)); -} - -function parseUsagePayload(payload) { - if (!payload || typeof payload !== "object") { - throw new Error("agent.usage.list returned an empty payload"); - } - - const source = payload.usage || payload.tokens || payload; - if (!hasUsageFields(source)) { - throw new Error("agent.usage.list did not return usage fields"); - } - - return normalizeUsage(payload); -} - -function isRefreshingUsageCacheStatus(cacheStatus) { - return Boolean(cacheStatus && typeof cacheStatus === "object" && cacheStatus.status === "refreshing"); -} - -function shouldPreserveExistingUsageSnapshot(row, snapshot, cacheStatus) { - if (!isRefreshingUsageCacheStatus(cacheStatus)) { - return false; - } - - const previousUsage = normalizeUsage(jsonParse(row?.usage_snapshot_json, {}).usage); - const nextUsage = normalizeUsage(snapshot?.usage); - return Number(previousUsage.daily) > 0 && Number(nextUsage.daily) <= 0; -} - function firstText(...values) { for (const value of values) { if (typeof value === "string" && value.trim()) { @@ -209,55 +165,6 @@ function readCatalogItems(snapshot, kind) { return catalog.items; } -function normalizeUsageItemPayload(item) { - if (!item || typeof item !== "object") { - return null; - } - - const agentId = firstText(item?.agentId, item?.agent_id, item?.id, item?.slug); - if (!agentId) { - return null; - } - - return { - agentId, - raw: item, - usage: parseUsagePayload(item), - captured_at: toChinaISOString() - }; -} - -function normalizeGatewayUsageItemPayload(item) { - const snapshot = normalizeUsageItemPayload(item); - if (!snapshot) { - return null; - } - - const source = item?.usage || item?.tokens || item || {}; - const usage = { - daily: Number(source.daily ?? source.day ?? source.dailyTokens ?? source.today ?? source.totalTokens ?? 0), - weekly: Number(source.weekly ?? source.weeklyTokens ?? 0), - monthly: Number(source.monthly ?? source.monthlyTokens ?? 0), - total: Number(source.total ?? source.totalTokens ?? 0) - }; - - return { - ...snapshot, - usage, - raw: { - ...snapshot.raw, - source: firstText(item?.source, "model-gateway"), - senders: Array.isArray(item?.senders) ? item.senders : [] - } - }; -} - -function extractUsageCacheStatus(payload) { - return payload && typeof payload === "object" && payload.cacheStatus && typeof payload.cacheStatus === "object" - ? payload.cacheStatus - : null; -} - function countLargeLanguageModels(items) { return items.reduce((count, provider) => count + (Array.isArray(provider?.models) ? provider.models.length : 0), 0); } @@ -270,13 +177,11 @@ function builtInOntologyNames(items) { } export class CatalogSyncService { - constructor({ db, managementClient, auditLogService = null, env = null, usageService = null }) { + constructor({ db, managementClient, auditLogService = null, env = null }) { this.db = db; this.managementClient = managementClient; - this.usageService = usageService; this.auditLogService = auditLogService; this.managementTimeoutMs = Number(env?.managementTimeoutMs || 120000); - this.pendingUsageRefresh = null; this.pendingLlmSync = null; this.pendingAgentSync = null; this.pendingSkillSync = null; @@ -286,10 +191,6 @@ export class CatalogSyncService { this.pendingCatalogSyncPlan = null; } - getUsageRefreshStatus() { - return this.readState("usage_refresh_state"); - } - getAgentSyncStatus() { return this.readState("agent_sync_state"); } @@ -339,11 +240,7 @@ export class CatalogSyncService { this.pendingCatalogSyncPlan = (async () => { const catalog = await this.syncCatalogTask({ trigger }); - const usage = await this.refreshAgentUsage({ trigger }); - return { - catalog, - usage - }; + return { catalog }; })().finally(() => { this.pendingCatalogSyncPlan = null; }); @@ -429,7 +326,7 @@ export class CatalogSyncService { await withSerializedTransaction(this.db, () => { this.syncLargeLanguageModels(llmItems, now); this.syncTemplates(templateItems, now); - this.syncAgents(agentItems, new Map(), now); + this.syncAgents(agentItems, now); this.syncSkills(skillItems, now); this.syncSystemBuiltInFlags(builtInOntologyNames(ontologyItems), now); @@ -485,160 +382,10 @@ export class CatalogSyncService { } } - async fetchAgentUsageSnapshots(agentItems) { - const agentIds = agentItems.map((item) => String(item?.id || "").trim()).filter(Boolean); - if (this.usageService && typeof this.usageService.listAgentUsage === "function") { - const remote = this.usageService.listAgentUsage({ agentIds }); - const pairs = listRpcItems(remote, "model-gateway.usage.list") - .map((item) => normalizeGatewayUsageItemPayload(item)) - .filter(Boolean) - .map((snapshot) => [snapshot.agentId, { - raw: snapshot.raw, - usage: snapshot.usage, - source: snapshot.raw.source, - senders: snapshot.raw.senders, - captured_at: snapshot.captured_at - }]); - return { - cacheStatus: null, - usageMap: new Map(pairs) - }; - } - const remote = await this.callManagement("agent.usage.list", { agentIds }); - const pairs = listRpcItems(remote, "agent.usage.list") - .map((item) => normalizeUsageItemPayload(item)) - .filter(Boolean) - .map((snapshot) => [snapshot.agentId, { - raw: snapshot.raw, - usage: snapshot.usage, - captured_at: snapshot.captured_at - }]); - return { - cacheStatus: extractUsageCacheStatus(remote), - usageMap: new Map(pairs) - }; - } - - async refreshAgentUsage({ trigger = "scheduled" } = {}) { - if (this.pendingUsageRefresh) { - return await this.pendingUsageRefresh; - } - - this.pendingUsageRefresh = this.runUsageRefresh(trigger).finally(() => { - this.pendingUsageRefresh = null; - }); - - return await this.pendingUsageRefresh; - } - - async runUsageRefresh(trigger) { - const startedAt = toChinaISOString(); - - const managementConfigured = typeof this.managementClient?.isConfigured === "function" - && this.managementClient.isConfigured(); - if (!managementConfigured && !this.usageService) { - this.updateState("usage_refresh_state", { - status: "skipped", - triggerSource: trigger, - startedAt, - finishedAt: startedAt, - lastSuccessAt: null, - errorMessage: "Management action is not configured", - summary: { - refreshed_agents: 0 - } - }); - return this.getUsageRefreshStatus(); - } - - this.updateState("usage_refresh_state", { - status: "running", - triggerSource: trigger, - startedAt, - finishedAt: null, - errorMessage: null, - summary: {} - }); - - const agentRows = this.db.prepare(` - SELECT * - FROM agents - ORDER BY slug - `).all(); - - if (agentRows.length === 0) { - this.updateState("usage_refresh_state", { - status: "skipped", - triggerSource: trigger, - startedAt, - finishedAt: startedAt, - lastSuccessAt: null, - errorMessage: null, - summary: {} - }); - return this.getUsageRefreshStatus(); - } - - const remoteItems = agentRows.map((row) => ({ - agentId: row.slug - })); - const capturedAt = toChinaISOString(); - const updateAgent = this.db.prepare(` - UPDATE agents - SET usage_snapshot_json = ?, status = ?, updated_at = ? - WHERE id = ? - `); - - try { - const { cacheStatus, usageMap } = await this.fetchAgentUsageSnapshots(remoteItems); - let refreshedAgents = 0; - - await withSerializedTransaction(this.db, () => { - for (const row of agentRows) { - const snapshot = usageMap.get(row.slug); - if (!snapshot) { - continue; - } - if (shouldPreserveExistingUsageSnapshot(row, snapshot, cacheStatus)) { - continue; - } - - const nextStatus = statusAfterUsageRefresh(row, snapshot.usage); - updateAgent.run(JSON.stringify(snapshot), nextStatus, capturedAt, row.id); - refreshedAgents += 1; - } - - const finishedAt = toChinaISOString(); - this.updateState("usage_refresh_state", { - status: "success", - triggerSource: trigger, - startedAt, - finishedAt, - lastSuccessAt: finishedAt, - errorMessage: null, - summary: { - refreshed_agents: refreshedAgents, - captured_at: capturedAt - } - }); - }); - - return this.getUsageRefreshStatus(); - } catch (error) { - const finishedAt = toChinaISOString(); - this.updateState("usage_refresh_state", { - status: "failed", - triggerSource: trigger, - startedAt, - finishedAt, - errorMessage: error instanceof Error ? error.message : String(error), - summary: {} - }); - return this.getUsageRefreshStatus(); - } - } - - syncAgents(items, usageMap, now) { + syncAgents(items, nowOrLegacyUsageMap, maybeNow) { + // The second argument used to be a usage snapshot map. Accept it for + // database upgrade compatibility, but deliberately ignore it. + const now = maybeNow || nowOrLegacyUsageMap || toChinaISOString(); const existingRows = this.db.prepare("SELECT * FROM agents").all(); const existingMap = new Map(existingRows.map((row) => [row.slug, row])); const seen = new Set(); @@ -656,7 +403,7 @@ export class CatalogSyncService { SET agent_name = ?, template_name = ?, llm_provider_id = ?, llm_model_id = ?, llm_model_ref = ?, fallback_llm_provider_id = ?, fallback_llm_model_id = ?, fallback_llm_model_ref = ?, - status = ?, usage_snapshot_json = ?, remote_state_json = ?, updated_at = ? + status = ?, remote_state_json = ?, updated_at = ? WHERE id = ? `); const remove = this.db.prepare("DELETE FROM agents WHERE id = ?"); @@ -679,13 +426,6 @@ export class CatalogSyncService { seen.add(slug); const existing = existingMap.get(slug); - const existingUsageSnapshot = jsonParse(existing?.usage_snapshot_json, { - usage: { daily: 0 } - }); - const usageSnapshot = usageMap.get(slug) || { - ...existingUsageSnapshot, - usage: normalizeUsage(existingUsageSnapshot.usage) - }; const remoteStateJson = JSON.stringify(normalizeAgentRemoteState(item)); const agentName = firstText(item?.name, slug); const status = normalizeAgentStatus(item?.status); @@ -706,7 +446,6 @@ export class CatalogSyncService { hasFallbackModelPayload ? model.fallbackModelId || null : existing.fallback_llm_model_id || null, hasFallbackModelPayload ? model.fallbackModelRef || null : existing.fallback_llm_model_ref || null, nextStatus, - JSON.stringify(usageSnapshot), remoteStateJson, now, existing.id @@ -729,7 +468,7 @@ export class CatalogSyncService { status, "[]", -1, - JSON.stringify(usageSnapshot), + "{}", remoteStateJson, now, now diff --git a/packages/daedalus-agent-management-web/server/src/services/model-gateway-usage-service.js b/packages/daedalus-agent-management-web/server/src/services/model-gateway-usage-service.js index 47e9a74..99cac7c 100644 --- a/packages/daedalus-agent-management-web/server/src/services/model-gateway-usage-service.js +++ b/packages/daedalus-agent-management-web/server/src/services/model-gateway-usage-service.js @@ -5,6 +5,7 @@ import { chinaDateKey, toChinaISOString } from "../utils/china-time.js"; const GROUP_BY_VALUES = new Set(["agent", "sender", "agent_sender", "model", "day", "hour"]); const DATE_PATTERN = /^\d{4}-\d{2}-\d{2}$/; const MAX_SENDER_ID_LENGTH = 512; +const BILLABLE_USAGE_FILTER = "billable = 1 AND NOT (agent_id = 'runtime' AND sender_id = 'unknown')"; function normalizeText(value, field, { required = false, maxLength = 512 } = {}) { const normalized = String(value ?? "").trim(); @@ -148,6 +149,7 @@ export class ModelGatewayUsageService { required: true, maxLength: MAX_SENDER_ID_LENGTH }); + const runtimeObservation = agentId === "runtime" && senderId === "unknown"; const promptTokens = normalizeTokens(payload.promptTokens ?? payload.prompt_tokens); const completionTokens = normalizeTokens(payload.completionTokens ?? payload.completion_tokens); const totalTokens = normalizeTokens( @@ -180,7 +182,7 @@ export class ModelGatewayUsageService { cachedTokens, totalTokens, normalizeText(payload.usageSource || payload.usage_source, "usageSource") || "provider", - payload.billable === false ? 0 : 1, + payload.billable === false || runtimeObservation ? 0 : 1, payload.success === false ? 0 : 1, normalizeText(payload.errorCode || payload.error_code, "errorCode") || null, payload.latencyMs === undefined && payload.latency_ms === undefined @@ -212,7 +214,7 @@ export class ModelGatewayUsageService { day: "substr(created_at, 1, 10)", hour: "substr(created_at, 1, 13)" }[groupBy]; - const filters = ["created_at >= ?", "created_at < ?", "billable = 1"]; + const filters = ["created_at >= ?", "created_at < ?", BILLABLE_USAGE_FILTER]; const params = [startAt, endAt]; if (agentId !== undefined && String(agentId).trim()) { filters.push("agent_id = ?"); @@ -275,7 +277,7 @@ export class ModelGatewayUsageService { COALESCE(SUM(cached_tokens), 0) AS cached_tokens, COALESCE(SUM(total_tokens), 0) AS total_tokens FROM model_usage_events - WHERE created_at >= ? AND created_at < ? AND billable = 1${whereIds} + WHERE created_at >= ? AND created_at < ? AND ${BILLABLE_USAGE_FILTER}${whereIds} GROUP BY agent_id, sender_id `).all(...params); const aggregate = (fromDate, toDate) => { @@ -286,7 +288,7 @@ export class ModelGatewayUsageService { return this.db.prepare(` SELECT agent_id, COALESCE(SUM(total_tokens), 0) AS total_tokens FROM model_usage_events - WHERE created_at >= ? AND created_at < ? AND billable = 1${whereIds} + WHERE created_at >= ? AND created_at < ? AND ${BILLABLE_USAGE_FILTER}${whereIds} GROUP BY agent_id `).all(...values); }; diff --git a/packages/daedalus-agent-management-web/server/src/services/portal-service.js b/packages/daedalus-agent-management-web/server/src/services/portal-service.js index 7f71a2b..8bd06a4 100644 --- a/packages/daedalus-agent-management-web/server/src/services/portal-service.js +++ b/packages/daedalus-agent-management-web/server/src/services/portal-service.js @@ -29,6 +29,7 @@ const PACKAGE_ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), const APPS_ROOT = path.resolve(PACKAGE_ROOT, ".."); const REPO_ROOT = path.resolve(PACKAGE_ROOT, "..", ".."); const DEFAULT_APPS_HOME = "/opt/daedalus-agent-apps"; +const BILLABLE_USAGE_FILTER = "billable = 1 AND NOT (agent_id = 'runtime' AND sender_id = 'unknown')"; function hasTable(db, tableName) { try { @@ -476,6 +477,7 @@ export class PortalService { this.versionInfoCache = buildVersionInfoCache(this.versionInfoEntries); this.localVersionCacheLoaded = false; this.kernelVersionCacheLoaded = false; + this.versionInfoRefreshPromise = null; } ensureTemplateFallbackLlmColumns() { @@ -575,30 +577,45 @@ export class PortalService { } async refreshVersionInfoCache({ timeoutMs = 15000, force = false } = {}) { - if (this.kernelVersionCacheLoaded && !force) { - return this.getVersionInfo(); + if (this.versionInfoRefreshPromise) { + return await this.versionInfoRefreshPromise; } - if ( - typeof this.managementClient?.call !== "function" - || (typeof this.managementClient?.isConfigured === "function" && !this.managementClient.isConfigured()) - ) { + if (this.kernelVersionCacheLoaded && !force) { return this.getVersionInfo(); } + const refreshPromise = (async () => { + if ( + typeof this.managementClient?.call !== "function" + || (typeof this.managementClient?.isConfigured === "function" && !this.managementClient.isConfigured()) + ) { + return this.getVersionInfo(); + } + + try { + const remote = await this.managementClient.call("version.list", {}, timeoutMs); + const remoteItems = Array.isArray(remote?.items) + ? remote.items.map((item) => normalizeVersionItem(item, "runtime")) + : []; + this.writeVersionInfoCache(remoteItems, "management-actions"); + this.kernelVersionCacheLoaded = true; + return this.getVersionInfo(); + } catch (error) { + return { + ...this.getVersionInfo(), + error_message: getErrorMessage(error, "version.list failed") + }; + } + })(); + this.versionInfoRefreshPromise = refreshPromise; + try { - const remote = await this.managementClient.call("version.list", {}, timeoutMs); - const remoteItems = Array.isArray(remote?.items) - ? remote.items.map((item) => normalizeVersionItem(item, "runtime")) - : []; - this.writeVersionInfoCache(remoteItems, "management-actions"); - this.kernelVersionCacheLoaded = true; - return this.getVersionInfo(); - } catch (error) { - return { - ...this.getVersionInfo(), - error_message: getErrorMessage(error, "version.list failed") - }; + return await refreshPromise; + } finally { + if (this.versionInfoRefreshPromise === refreshPromise) { + this.versionInfoRefreshPromise = null; + } } } @@ -1413,21 +1430,12 @@ export class PortalService { FROM agent_sync_state WHERE id = 1 `).get(); - const usageRefreshState = this.db.prepare(` - SELECT status, last_success_at - FROM usage_refresh_state - WHERE id = 1 - `).get(); const agentStatsReady = Boolean(agentSyncState?.last_success_at) && agentSyncState?.status === "success"; - const usageStatsReady = Boolean(usageRefreshState?.last_success_at) && usageRefreshState?.status === "success"; const agentCount = this.db.prepare("SELECT COUNT(*) AS count FROM agents").get().count; const templateCount = this.db.prepare("SELECT COUNT(*) AS count FROM agent_templates").get().count; const skillCount = this.db.prepare("SELECT COUNT(*) AS count FROM skills").get().count; const systemCount = this.db.prepare("SELECT COUNT(*) AS count FROM business_systems").get().count; - let totalTokenUsage = Number(this.db.prepare(` - SELECT COALESCE(SUM(COALESCE(json_extract(usage_snapshot_json, '$.usage.daily'), 0)), 0) AS total - FROM agents - `).get()?.total || 0); + let totalTokenUsage = 0; const usageLedgerReady = hasTable(this.db, "model_usage_events"); const today = chinaDateKey(); const tomorrow = addChinaDays(today, 1); @@ -1435,7 +1443,7 @@ export class PortalService { totalTokenUsage = Number(this.db.prepare(` SELECT COALESCE(SUM(total_tokens), 0) AS total FROM model_usage_events - WHERE created_at >= ? AND created_at < ? AND billable = 1 + WHERE created_at >= ? AND created_at < ? AND ${BILLABLE_USAGE_FILTER} `).get(`${today}T00:00:00.000+08:00`, `${tomorrow}T00:00:00.000+08:00`)?.total || 0); } const recentInvocations = this.db.prepare(` @@ -1461,25 +1469,12 @@ export class PortalService { ON events.agent_id = ag.slug AND events.created_at >= ? AND events.created_at < ? - AND events.billable = 1 + AND ${BILLABLE_USAGE_FILTER} GROUP BY ag.id, ag.slug, ag.agent_name, ag.status, ag.daily_limit HAVING today_tokens > 0 ORDER BY today_tokens DESC, ag.slug ASC - LIMIT 50 `).all(`${today}T00:00:00.000+08:00`, `${tomorrow}T00:00:00.000+08:00`) - : this.db.prepare(` - SELECT - id, - slug, - agent_name, - status, - daily_limit, - COALESCE(json_extract(usage_snapshot_json, '$.usage.daily'), 0) AS today_tokens - FROM agents - WHERE COALESCE(json_extract(usage_snapshot_json, '$.usage.daily'), 0) > 0 - ORDER BY today_tokens DESC, updated_at DESC, id DESC - LIMIT 50 - `).all(); + : []; const agentUsage = agentUsageRows.map((row) => { const dailyLimit = Number(row.daily_limit ?? -1); const todayTokens = Number(row.today_tokens ?? 0); @@ -1505,11 +1500,10 @@ export class PortalService { ON events.sender_id = users.username AND events.created_at >= ? AND events.created_at < ? - AND events.billable = 1 + AND ${BILLABLE_USAGE_FILTER} GROUP BY users.id, users.username, users.daily_limit HAVING today_tokens > 0 ORDER BY today_tokens DESC, users.username ASC - LIMIT 50 `).all(`${today}T00:00:00.000+08:00`, `${tomorrow}T00:00:00.000+08:00`) : []; const normalizedUserUsage = userUsage.map((row) => { @@ -1529,7 +1523,7 @@ export class PortalService { templates: templateCount, skills: skillCount, systems: systemCount, - today_tokens: usageLedgerReady || usageStatsReady ? totalTokenUsage : null + today_tokens: totalTokenUsage }, recentInvocations, agentUsage, diff --git a/packages/daedalus-agent-management-web/server/test/agent-service-alignment.test.js b/packages/daedalus-agent-management-web/server/test/agent-service-alignment.test.js index db1cec4..273a1de 100644 --- a/packages/daedalus-agent-management-web/server/test/agent-service-alignment.test.js +++ b/packages/daedalus-agent-management-web/server/test/agent-service-alignment.test.js @@ -127,6 +127,42 @@ it("reports normal health for normal agents", async () => { expect(agent.health).toBe("normal"); }); +it("reads Agent usage directly from the Model Gateway ledger service", async () => { + const service = new AgentService({ + db: createDb({ + id: 2, + slug: "ledger-agent", + agent_name: "Ledger Agent", + description: "demo", + docs_content: "", + template_name: "default", + status: "normal", + tags_json: "[]", + daily_limit: 10, + usage_snapshot_json: JSON.stringify({ usage: { daily: 1 } }), + remote_state_json: "{}", + created_at: "2026-05-26T00:00:00.000Z", + updated_at: "2026-05-26T00:00:00.000Z" + }), + managementClient: {}, + objectStorage: {}, + usageService: { + listAgentUsage() { + return { + items: [{ + agentId: "ledger-agent", + usage: { daily: 12, weekly: 30, monthly: 40, total: 50 } + }] + }; + } + } + }); + + const [agent] = await service.listAgents(); + expect(agent.usage).toEqual({ daily: 12, weekly: 30, monthly: 40, total: 50 }); + expect(agent.health).toBe("overlimit"); +}); + it("exposes MQTT topics and the synchronous HTTP endpoint for each agent", async () => { const service = new AgentService({ db: createDb({ diff --git a/packages/daedalus-agent-management-web/server/test/app-smoke.test.js b/packages/daedalus-agent-management-web/server/test/app-smoke.test.js index daf1441..f7049f3 100644 --- a/packages/daedalus-agent-management-web/server/test/app-smoke.test.js +++ b/packages/daedalus-agent-management-web/server/test/app-smoke.test.js @@ -129,9 +129,7 @@ it("supports anonymous public external agent directories and id identity aliases it("handles synchronous agent requests and returns only the final result", async () => { const { app, services } = createServerApp(); const calls = []; - const usageRefreshes = []; const ensuredUserBatches = []; - let usageRefreshResult = { status: "success" }; services.externalService.assertAgentCallAvailable = () => {}; const originalEnsureAgentUsers = services.agentService.ensureAgentUsers.bind(services.agentService); services.agentService.ensureAgentUsers = (usernames, dailyLimit) => { @@ -146,13 +144,6 @@ it("handles synchronous agent requests and returns only the final result", async text: "同步结果" }; }; - services.catalogSyncService.refreshAgentUsage = async (options) => { - usageRefreshes.push(options); - if (usageRefreshResult === "throw") { - throw new Error("usage refresh unavailable"); - } - return usageRefreshResult; - }; const server = await listen(app); const baseUrl = `http://127.0.0.1:${server.address().port}`; @@ -189,7 +180,6 @@ it("handles synchronous agent requests and returns only the final result", async .toEqual({ daily_limit: -1 }); expect(services.agentService.db.prepare("SELECT id FROM users WHERE username = ?").get("sender-1")) .toBeUndefined(); - expect(usageRefreshes).toEqual([{ trigger: "external-agent-call" }]); const invalidResponse = await fetch(`${baseUrl}/api/external/agents/finance-agent/call`, { method: "POST", @@ -197,9 +187,6 @@ it("handles synchronous agent requests and returns only the final result", async body: JSON.stringify({ senderId: "sender-1" }) }); expect(invalidResponse.status).toBe(400); - expect(usageRefreshes).toHaveSize(1); - - usageRefreshResult = { status: "failed", error_message: "usage source unavailable" }; const failedRefreshResponse = await fetch(`${baseUrl}/api/external/agents/finance-agent/call`, { method: "POST", headers: { "content-type": "application/json" }, @@ -212,7 +199,6 @@ it("handles synchronous agent requests and returns only the final result", async text: "同步结果" }); - usageRefreshResult = "throw"; const thrownRefreshResponse = await fetch(`${baseUrl}/api/external/agents/finance-agent/call`, { method: "POST", headers: { "content-type": "application/json" }, @@ -224,7 +210,6 @@ it("handles synchronous agent requests and returns only the final result", async kind: "final", text: "同步结果" }); - expect(usageRefreshes).toHaveSize(3); expect(ensuredUserBatches).toEqual([ { usernames: ["sender-1"], dailyLimit: -1 }, { usernames: ["sender-1"], dailyLimit: -1 }, diff --git a/packages/daedalus-agent-management-web/server/test/background-services.test.js b/packages/daedalus-agent-management-web/server/test/background-services.test.js index 8b9af94..f3dbc59 100644 --- a/packages/daedalus-agent-management-web/server/test/background-services.test.js +++ b/packages/daedalus-agent-management-web/server/test/background-services.test.js @@ -78,13 +78,12 @@ function createServices({ catalogSyncService: { syncStartupCatalogTask: taskFactory("sync-startup-catalog"), syncCatalogTask: taskFactory("sync-catalog"), - refreshAgentUsage: taskFactory("usage"), markStartupSyncFailed } }; } -it("schedules catalog and token sync from completion times after startup catalog sync succeeds", async () => { +it("schedules only catalog sync from completion time after startup catalog sync succeeds", async () => { const timers = installFakeTimers(); const calls = []; const taskFactory = (label) => async ({ trigger }) => { @@ -113,28 +112,12 @@ it("schedules catalog and token sync from completion times after startup catalog "action.start", "sync-startup-catalog:startup" ]); - expect(activeDelays(timers)).toEqual([ - 60 * 1000, - 60 * 1000, - 2 * 60 * 1000 - ]); - - await timers.runTimer(timers.activeTimeouts()[1]); - expect(calls).toContain("usage:scheduled"); - expect(activeDelays(timers)).toEqual([ - 60 * 1000, - 2 * 60 * 1000, - 2 * 60 * 1000 - ]); + expect(activeDelays(timers)).toEqual([60 * 1000, 2 * 60 * 1000]); const catalogTimer = findActiveTimerByDelay(timers, 2 * 60 * 1000); await timers.runTimer(catalogTimer); expect(calls.filter((entry) => entry === "sync-catalog:scheduled").length).toBe(1); - expect(activeDelays(timers)).toEqual([ - 60 * 1000, - 2 * 60 * 1000, - 2 * 60 * 1000 - ]); + expect(activeDelays(timers)).toEqual([60 * 1000, 2 * 60 * 1000]); } finally { await background.stop(); timers.restore(); @@ -183,11 +166,7 @@ it("refreshes system status one minute after startup and then every minute", asy ] } }); - expect(activeDelays(timers)).toEqual([ - 60 * 1000, - 60 * 1000, - 2 * 60 * 1000 - ]); + expect(activeDelays(timers)).toEqual([60 * 1000, 2 * 60 * 1000]); } finally { await background.stop(); timers.restore(); @@ -218,8 +197,6 @@ it("keeps successful scheduled background work quiet", async () => { const output = stderrWrite.calls.allArgs().map(([value]) => String(value)).join("\n"); expect(output).not.toContain("System status refresh starting"); expect(output).not.toContain("System status refresh finished"); - expect(output).not.toContain("Scheduled agent usage sync starting"); - expect(output).not.toContain("Scheduled agent usage sync finished"); expect(output).not.toContain("Scheduled catalog sync starting"); expect(output).not.toContain("Scheduled catalog sync finished"); } finally { @@ -254,7 +231,6 @@ it("still logs scheduled background failures", async () => { const output = stderrWrite.calls.allArgs().map(([value]) => String(value)).join("\n"); expect(output).toContain("System status refresh target failed"); - expect(output).toContain("Scheduled agent usage sync failed"); expect(output).toContain("Scheduled catalog sync failed"); } finally { await background.stop(); @@ -262,7 +238,7 @@ it("still logs scheduled background failures", async () => { } }); -it("does not schedule token sync when startup catalog sync fails", async () => { +it("keeps catalog scheduling when startup catalog sync fails", async () => { spyOn(console, "error").and.stub(); const timers = installFakeTimers(); @@ -294,10 +270,7 @@ it("does not schedule token sync when startup catalog sync fails", async () => { "action.start", "sync-startup-catalog:startup" ]); - expect(activeDelays(timers)).toEqual([ - 60 * 1000, - 2 * 60 * 1000 - ]); + expect(activeDelays(timers)).toEqual([60 * 1000, 2 * 60 * 1000]); } finally { await background.stop(); timers.restore(); @@ -403,11 +376,7 @@ it("retries management action readiness before startup sync failure is marked", "sync-startup-catalog:startup" ]); expect(startupFailures).toEqual([]); - expect(activeDelays(timers)).toEqual([ - 60 * 1000, - 60 * 1000, - 2 * 60 * 1000 - ]); + expect(activeDelays(timers)).toEqual([60 * 1000, 2 * 60 * 1000]); } finally { await background.stop(); timers.restore(); diff --git a/packages/daedalus-agent-management-web/server/test/catalog-sync-route.test.js b/packages/daedalus-agent-management-web/server/test/catalog-sync-route.test.js index 26b4b7f..92848e5 100644 --- a/packages/daedalus-agent-management-web/server/test/catalog-sync-route.test.js +++ b/packages/daedalus-agent-management-web/server/test/catalog-sync-route.test.js @@ -60,7 +60,6 @@ it("runs the manual sync plan and returns all catalog sync states", async () => skill_sync: { status: "success", summary: { skills: 4 } }, template_sync: { status: "success", summary: { templates: 5 } }, system_sync: { status: "success", summary: { systems: 6 } }, - usage_refresh: { status: "skipped", summary: { refreshed_agents: 0 } } }; const app = express(); app.use(express.json()); @@ -88,10 +87,6 @@ it("runs the manual sync plan and returns all catalog sync states", async () => status: "success", trigger_source: trigger }, - usage: { - status: "success", - trigger_source: trigger - } }; }, getAgentSyncStatus: () => states.agent_sync, @@ -99,7 +94,6 @@ it("runs the manual sync plan and returns all catalog sync states", async () => getSkillSyncStatus: () => states.skill_sync, getTemplateSyncStatus: () => states.template_sync, getSystemSyncStatus: () => states.system_sync, - getUsageRefreshStatus: () => states.usage_refresh }, portalService: {}, env: {} @@ -119,14 +113,12 @@ it("runs the manual sync plan and returns all catalog sync states", async () => expect(response.status).toBe(200); expect(calls).toEqual([{ trigger: "manual" }]); expect(response.body.result.catalog.status).toBe("success"); - expect(response.body.result.usage.status).toBe("success"); expect(response.body.agent_sync).toEqual(states.agent_sync); - expect(response.body.usage_refresh).toEqual(states.usage_refresh); expect(auditWrites).toEqual([{ userId: 1, username: "daedalus-agent", - action: "同步数据", - detail: "手动触发 catalog.snapshot 与 Model Gateway 用量快照同步" + action: "同步目录", + detail: "手动触发 catalog.snapshot，同步目录数据" }]); } finally { await close(server); diff --git a/packages/daedalus-agent-management-web/server/test/catalog-sync-service.test.js b/packages/daedalus-agent-management-web/server/test/catalog-sync-service.test.js index 61f171e..aba4eb7 100644 --- a/packages/daedalus-agent-management-web/server/test/catalog-sync-service.test.js +++ b/packages/daedalus-agent-management-web/server/test/catalog-sync-service.test.js @@ -1,7 +1,6 @@ import { DatabaseSync } from "node:sqlite"; import { CatalogSyncService } from "../src/services/catalog-sync-service.js"; -import { ModelGatewayUsageService } from "../src/services/model-gateway-usage-service.js"; const TEST_NOW = "2026-05-26T00:00:00.000Z"; @@ -613,8 +612,6 @@ it("syncs startup catalog snapshot with one management action runtime call", asy expect(service.getAgentSyncStatus().summary).toEqual({ agents: 1 }); expect(service.getSkillSyncStatus().summary).toEqual({ skills: 1 }); expect(service.getSystemSyncStatus().summary).toEqual({ systems: 2 }); - expect(service.getUsageRefreshStatus().status).toBe("idle"); - expect(service.getUsageRefreshStatus().summary).toEqual({}); }); it("clears stale fallback model when the catalog explicitly omits fallbacks", () => { @@ -715,7 +712,7 @@ it("keeps local catalog rows when startup catalog sync fails", async () => { } }); -it("runs the manual sync plan as catalog snapshot sync followed by token usage sync", async () => { +it("runs the manual sync plan as catalog snapshot sync only", async () => { const db = createDb(); insertAgent(db, { slug: "alpha", @@ -740,16 +737,6 @@ it("runs the manual sync plan as catalog snapshot sync followed by token usage s }] }); } - if (action === "agent.usage.list") { - return { - items: [{ - agentId: "alpha", - usage: { - daily: 9 - } - }] - }; - } throw new Error(`Unexpected action: ${action}`); } } @@ -758,106 +745,41 @@ it("runs the manual sync plan as catalog snapshot sync followed by token usage s const result = await service.runManualSyncPlan({ trigger: "manual" }); expect(result.catalog.status).toBe("success"); - expect(result.usage.status).toBe("success"); - expect(calls).toEqual([ - "catalog.snapshot", - "agent.usage.list" - ]); - expect(service.getUsageRefreshStatus().summary.refreshed_agents).toBe(1); + expect(result).toEqual(jasmine.objectContaining({ + catalog: jasmine.objectContaining({ status: "success" }) + })); + expect(calls).toEqual(["catalog.snapshot"]); }); - -it("refreshes usage snapshots from the Model Gateway ledger without calling the kernel usage RPC", async () => { +it("does not call usage RPCs or mutate legacy usage snapshots during catalog sync", async () => { const db = createDb(); + const legacySnapshot = JSON.stringify({ usage: { daily: 42 }, captured_at: "legacy" }); insertAgent(db, { slug: "alpha", - agentName: "Alpha" - }); - const usageService = new ModelGatewayUsageService({ - db, - now: () => new Date("2026-05-26T00:00:00.000Z") - }); - usageService.recordUsageEvent({ - eventId: "gateway-evt-1", - requestId: "gateway-req-1", - agentId: "alpha", - senderId: "alice", - promptTokens: 10, - completionTokens: 5, - createdAt: "2026-05-26T10:00:00.000+08:00" + agentName: "Alpha", + usageSnapshotJson: legacySnapshot }); - const calls = []; const service = new CatalogSyncService({ db, - usageService, managementClient: { - isConfigured() { - return true; - }, + isConfigured: () => true, async call(action) { calls.push(action); - if (action === "catalog.snapshot") { - return catalogSnapshot({ - agents: [{ id: "alpha", name: "Alpha", status: "active" }] - }); + if (action !== "catalog.snapshot") { + throw new Error(`Unexpected action: ${action}`); } - throw new Error(`Unexpected action: ${action}`); + return catalogSnapshot({ agents: [{ id: "alpha", name: "Alpha", status: "active" }] }); } } }); - const result = await service.runManualSyncPlan({ trigger: "manual" }); + await service.runManualSyncPlan({ trigger: "manual" }); - expect(result.usage.status).toBe("success"); expect(calls).toEqual(["catalog.snapshot"]); - const snapshot = JSON.parse(db.prepare("SELECT usage_snapshot_json FROM agents WHERE slug = ?").get("alpha").usage_snapshot_json); - expect(snapshot.source).toBe("model-gateway"); - expect(snapshot.usage).toEqual({ daily: 15, weekly: 15, monthly: 15, total: 15 }); - expect(snapshot.senders).toEqual([{ - senderId: "alice", - requests: 1, - promptTokens: 10, - completionTokens: 5, - cachedTokens: 0, - totalTokens: 15 - }]); + expect(db.prepare("SELECT usage_snapshot_json FROM agents WHERE slug = ?").get("alpha").usage_snapshot_json) + .toBe(legacySnapshot); }); -it("refreshes usage snapshots from the Gateway ledger when the management runtime is not configured", async () => { - const db = createDb(); - insertAgent(db, { - slug: "alpha", - agentName: "Alpha" - }); - const usageService = new ModelGatewayUsageService({ - db, - now: () => new Date("2026-05-26T00:00:00.000Z") - }); - usageService.recordUsageEvent({ - eventId: "gateway-unconfigured-event", - agentId: "alpha", - senderId: "alice", - totalTokens: 6, - createdAt: "2026-05-26T10:00:00.000+08:00" - }); - - const service = new CatalogSyncService({ - db, - usageService, - managementClient: { - isConfigured() { - return false; - } - } - }); - - const result = await service.refreshAgentUsage({ trigger: "scheduled" }); - - expect(result.status).toBe("success"); - expect(result.summary.refreshed_agents).toBe(1); - expect(JSON.parse(db.prepare("SELECT usage_snapshot_json FROM agents WHERE slug = ?").get("alpha").usage_snapshot_json).usage) - .toEqual({ daily: 6, weekly: 6, monthly: 6, total: 6 }); -}); it("uses kernel agent status as source while keeping local overlimit state", async () => { const db = createDb(); @@ -953,249 +875,6 @@ it("keeps renamed agent name from kernel catalog sync for management and externa }); }); -it("refreshes agent usage snapshots with kernel stock values", async () => { - const db = createDb(); - insertAgent(db, { - slug: "alpha", - agentName: "Alpha", - status: "active", - usageSnapshotJson: JSON.stringify({ - usage: { - daily: 1, - weekly: 2, - monthly: 3, - total: 10 - } - }) - }); - - const service = new CatalogSyncService({ - db, - auditLogService: { - write() {} - }, - managementClient: { - isConfigured() { - return true; - }, - async call(action) { - if (action !== "agent.usage.list") { - throw new Error(`Unexpected action: ${action}`); - } - - return { - items: [{ - agentId: "alpha", - usage: { - daily: 4 - } - }] - }; - } - } - }); - - const result = await service.refreshAgentUsage({ trigger: "scheduled" }); - - expect(result.status).toBe("success"); - expect(result.trigger_source).toBe("scheduled"); - expect(result.summary.refreshed_agents).toBe(1); - expect(result.summary.captured_at).toContain("T"); - - const agent = db.prepare("SELECT * FROM agents WHERE slug = ?").get("alpha"); - expect(JSON.parse(agent.usage_snapshot_json).usage).toEqual({ - daily: 4 - }); -}); - -it("overwrites agent usage snapshots with latest kernel stock values", async () => { - const db = createDb(); - insertAgent(db, { - slug: "alpha", - agentName: "Alpha" - }); - const usageValues = [4, 6]; - const service = new CatalogSyncService({ - db, - managementClient: { - isConfigured() { - return true; - }, - async call(action) { - if (action !== "agent.usage.list") { - throw new Error(`Unexpected action: ${action}`); - } - - return { - items: [{ - agentId: "alpha", - usage: { - daily: usageValues.shift() - } - }] - }; - } - } - }); - - await service.refreshAgentUsage({ trigger: "scheduled" }); - await service.refreshAgentUsage({ trigger: "scheduled" }); - - const agent = db.prepare("SELECT * FROM agents WHERE slug = ?").get("alpha"); - expect(JSON.parse(agent.usage_snapshot_json).usage).toEqual({ - daily: 6 - }); -}); - -it("keeps previous nonzero usage while kernel usage cache is refreshing with zero values", async () => { - const db = createDb(); - insertAgent(db, { - slug: "support", - agentName: "Support", - usageSnapshotJson: JSON.stringify({ - usage: { - daily: 12358608 - }, - captured_at: "2026-07-02T17:28:35.188+08:00" - }) - }); - const service = new CatalogSyncService({ - db, - managementClient: { - isConfigured() { - return true; - }, - async call(action) { - if (action !== "agent.usage.list") { - throw new Error(`Unexpected action: ${action}`); - } - - return { - cacheStatus: { - status: "refreshing", - cachedFiles: 0, - pendingFiles: 1, - staleFiles: 1 - }, - items: [{ - agentId: "support", - usage: { - daily: 0 - } - }] - }; - } - } - }); - - const result = await service.refreshAgentUsage({ trigger: "scheduled" }); - - expect(result.status).toBe("success"); - expect(result.summary.refreshed_agents).toBe(0); - - const agent = db.prepare("SELECT * FROM agents WHERE slug = ?").get("support"); - expect(JSON.parse(agent.usage_snapshot_json).usage).toEqual({ - daily: 12358608 - }); -}); - -it("does not write audit logs during agent usage refresh", async () => { - const db = createDb(); - insertAgent(db, { - slug: "alpha", - agentName: "Alpha", - status: "active" - }); - const auditWrites = []; - - const service = new CatalogSyncService({ - db, - auditLogService: { - write(entry) { - auditWrites.push(entry); - } - }, - managementClient: { - isConfigured() { - return true; - }, - async call(action) { - if (action !== "agent.usage.list") { - throw new Error(`Unexpected action: ${action}`); - } - - return { - items: [{ - agentId: "alpha", - usage: { - daily: 4 - } - }] - }; - } - } - }); - - const result = await service.refreshAgentUsage({ trigger: "scheduled" }); - - expect(result.status).toBe("success"); - expect(auditWrites).toEqual([]); -}); - -it("moves non-disabled agents to overlimit when refreshed usage reaches quota", async () => { - const db = createDb(); - insertAgent(db, { - slug: "alpha", - agentName: "Alpha", - dailyLimit: 10 - }); - insertAgent(db, { - slug: "beta", - agentName: "Beta", - status: "disabled", - dailyLimit: 10 - }); - - const service = new CatalogSyncService({ - db, - auditLogService: { - write() {} - }, - managementClient: { - isConfigured() { - return true; - }, - async call(action) { - if (action !== "agent.usage.list") { - throw new Error(`Unexpected action: ${action}`); - } - - return { - items: [ - { - agentId: "alpha", - usage: { - daily: 12 - } - }, - { - agentId: "beta", - usage: { - daily: 20 - } - } - ] - }; - } - } - }); - - const result = await service.refreshAgentUsage({ trigger: "scheduled" }); - - expect(result.status).toBe("success"); - expect(db.prepare("SELECT status FROM agents WHERE slug = ?").get("alpha").status).toBe("overlimit"); - expect(db.prepare("SELECT status FROM agents WHERE slug = ?").get("beta").status).toBe("disabled"); -}); it("records skipped status when management action runtime is not configured", async () => { const db = createDb(); @@ -1241,41 +920,4 @@ it("marks startup catalog sync states failed when management action runtime is n expect(row.error_message).toBe("Management action was not ready"); expect(row.finished_at).toBeDefined(); } - expect(db.prepare("SELECT status FROM usage_refresh_state WHERE id = 1").get().status).toBe("idle"); -}); - -it("fails usage refresh when agent.usage.list returns a non-usage payload", async () => { - const db = createDb(); - insertAgent(db, { - slug: "alpha", - agentName: "Alpha", - status: "active" - }); - - const service = new CatalogSyncService({ - db, - managementClient: { - isConfigured() { - return true; - }, - async call(action) { - if (action !== "agent.usage.list") { - throw new Error(`Unexpected action: ${action}`); - } - - return { - items: [{ - agentId: "alpha", - configPath: "/var/daedalus-agent/.openclaw/openclaw.json", - defaultModel: "corp-openai/gpt-5.4" - }] - }; - } - } - }); - - const result = await service.refreshAgentUsage({ trigger: "scheduled" }); - - expect(result.status).toBe("failed"); - expect(result.error_message).toContain("usage fields"); }); diff --git a/packages/daedalus-agent-management-web/server/test/management-action-client.test.js b/packages/daedalus-agent-management-web/server/test/management-action-client.test.js index 04621dd..809a2be 100644 --- a/packages/daedalus-agent-management-web/server/test/management-action-client.test.js +++ b/packages/daedalus-agent-management-web/server/test/management-action-client.test.js @@ -78,7 +78,7 @@ it("marks timed out in-process actions as failed", async () => { await new Promise((resolve) => setTimeout(resolve, 50)); return { requestId: "late", - action: "agent.usage.list", + action: "catalog.snapshot", ok: true, result: {} }; @@ -91,7 +91,7 @@ it("marks timed out in-process actions as failed", async () => { }); await expectAsync( - managementClient.call("agent.usage.list", { agentIds: ["alpha"] }) + managementClient.call("catalog.snapshot", {}) ).toBeRejectedWithError(/Management action timed out/); const failureUpdate = db.statements.find((item) => item.sql.includes("UPDATE management_requests")); @@ -99,7 +99,7 @@ it("marks timed out in-process actions as failed", async () => { const [errorJson] = failureUpdate.args; expect(JSON.parse(errorJson).details).toEqual(jasmine.objectContaining({ timeout: true, - action: "agent.usage.list" + action: "catalog.snapshot" })); }); diff --git a/packages/daedalus-agent-management-web/server/test/model-gateway-usage-route.test.js b/packages/daedalus-agent-management-web/server/test/model-gateway-usage-route.test.js index f3d5a22..5cfd0c0 100644 --- a/packages/daedalus-agent-management-web/server/test/model-gateway-usage-route.test.js +++ b/packages/daedalus-agent-management-web/server/test/model-gateway-usage-route.test.js @@ -45,6 +45,7 @@ function requestJson(url, { method, body, headers = {} } = {}) { function createApp({ remoteAddress = "" } = {}) { const db = new DatabaseSync(":memory:"); + const broadcasts = []; const usageService = new ModelGatewayUsageService({ db, now: () => new Date("2026-09-01T04:00:00.000Z") @@ -80,16 +81,21 @@ function createApp({ remoteAddress = "" } = {}) { }) }, auditLogService: { write() {} }, + updateNotifier: { + broadcast(type, payload) { + broadcasts.push({ type, payload }); + } + }, env: {} })); app.use((error, _req, res, _next) => { res.status(error.status || 500).json({ code: error.code || "internal_error", message: error.message }); }); - return { app, db }; + return { app, db, broadcasts }; } it("checks today's user and agent quotas for the OpenClaw run gate", async () => { - const { app, db } = createApp(); + const { app, db, broadcasts } = createApp(); const server = await listen(app); try { const baseUrl = `http://127.0.0.1:${server.address().port}`; @@ -114,7 +120,7 @@ it("checks today's user and agent quotas for the OpenClaw run gate", async () => }); it("writes idempotent gateway usage events and queries agent/sender dimensions", async () => { - const { app, db } = createApp(); + const { app, db, broadcasts } = createApp(); const server = await listen(app); try { const baseUrl = `http://127.0.0.1:${server.address().port}`; @@ -137,10 +143,15 @@ it("writes idempotent gateway usage events and queries agent/sender dimensions", const inserted = await requestJson(`${baseUrl}/api/internal/model-gateway/usage-events`, { method: "POST", body: event }); expect(inserted.status).toBe(201); expect(inserted.body).toEqual({ eventId: "route-event-1", inserted: true }); + expect(broadcasts).toEqual([{ + type: "usage.updated", + payload: { trigger: "model-gateway", event_id: "route-event-1" } + }]); const duplicate = await requestJson(`${baseUrl}/api/internal/model-gateway/usage-events`, { method: "POST", body: event }); expect(duplicate.status).toBe(200); expect(duplicate.body).toEqual({ eventId: "route-event-1", inserted: false }); + expect(broadcasts).toHaveSize(1); const second = await requestJson(`${baseUrl}/api/internal/model-gateway/usage-events`, { method: "POST", diff --git a/packages/daedalus-agent-management-web/server/test/model-gateway-usage-service.test.js b/packages/daedalus-agent-management-web/server/test/model-gateway-usage-service.test.js index 44305b6..8069dd3 100644 --- a/packages/daedalus-agent-management-web/server/test/model-gateway-usage-service.test.js +++ b/packages/daedalus-agent-management-web/server/test/model-gateway-usage-service.test.js @@ -77,6 +77,17 @@ it("records idempotent model usage events and aggregates by agent and sender", ( billable: false, createdAt: "2026-09-01T11:45:00.000+08:00" }); + service.recordUsageEvent({ + eventId: "evt-runtime-observation", + agentId: "runtime", + senderId: "unknown", + totalTokens: 32, + createdAt: "2026-09-01T11:46:00.000+08:00" + }); + + expect(db.prepare("SELECT billable FROM model_usage_events WHERE event_id = 'evt-runtime-observation'").get().billable).toBe(0); + // 旧版本可能已经把 Gateway 观测行写成 billable=1，查询仍需排除它。 + db.prepare("UPDATE model_usage_events SET billable = 1 WHERE event_id = 'evt-runtime-observation'").run(); expect(service.querySummary({ from: "2026-09-01", diff --git a/packages/daedalus-agent-management-web/server/test/portal-service-alignment.test.js b/packages/daedalus-agent-management-web/server/test/portal-service-alignment.test.js index d482876..310e991 100644 --- a/packages/daedalus-agent-management-web/server/test/portal-service-alignment.test.js +++ b/packages/daedalus-agent-management-web/server/test/portal-service-alignment.test.js @@ -158,7 +158,7 @@ function replaceZipEntryName(buffer, sourceName, targetName) { return result; } -it("returns dashboard placeholders before agent and usage startup sync complete", () => { +it("returns dashboard metrics from the ledger without usage startup sync state", () => { const scalarValues = new Map([ ["SELECT status, last_success_at FROM agent_sync_state WHERE id = 1", { status: "running", last_success_at: null }], ["SELECT status, last_success_at FROM usage_refresh_state WHERE id = 1", { status: "idle", last_success_at: null }], @@ -192,12 +192,12 @@ it("returns dashboard placeholders before agent and usage startup sync complete" templates: 2, skills: 3, systems: 4, - today_tokens: null + today_tokens: 0 }); expect(dashboard.agentUsage).toEqual([]); }); -it("returns dashboard metrics after agent and usage startup sync complete", () => { +it("does not read legacy usage snapshots when the ledger is unavailable", () => { const scalarValues = new Map([ ["SELECT status, last_success_at FROM agent_sync_state WHERE id = 1", { status: "success", last_success_at: "2026-05-27T08:00:00.000Z" }], ["SELECT status, last_success_at FROM usage_refresh_state WHERE id = 1", { status: "success", last_success_at: "2026-05-27T08:01:00.000Z" }], @@ -240,16 +240,9 @@ it("returns dashboard metrics after agent and usage startup sync complete", () = templates: 2, skills: 3, systems: 4, - today_tokens: 123 + today_tokens: 0 }); - expect(dashboard.agentUsage).toEqual([{ - id: 1, - slug: "finance-agent", - agent_name: "Finance Agent", - status: "normal", - daily_limit: -1, - today_tokens: 123 - }]); + expect(dashboard.agentUsage).toEqual([]); }); it("aggregates dashboard agent and user usage from today's billable gateway events only", () => { @@ -304,8 +297,13 @@ it("aggregates dashboard agent and user usage from today's billable gateway even INSERT INTO model_usage_events (event_id, agent_id, sender_id, total_tokens, billable, created_at) VALUES (?, ?, ?, ?, ?, ?) `).run("ignored", "idle-agent", "bob", 999, 0, today); + db.prepare(` + INSERT INTO model_usage_events (event_id, agent_id, sender_id, total_tokens, billable, created_at) + VALUES (?, ?, ?, ?, ?, ?) + `).run("gateway-observation", "runtime", "unknown", 100, 1, today); const dashboard = new PortalService({ db, objectStorage: {}, managementClient: {}, authService: {} }).getDashboard(); + expect(dashboard.stats.today_tokens).toBe(100); expect(dashboard.agentUsage).toEqual([jasmine.objectContaining({ slug: "finance-agent", today_tokens: 100, @@ -588,6 +586,40 @@ it("loads runtime versions once after management action runtime is ready", async }]); }); +it("shares an in-flight runtime version refresh across callers", async () => { + let resolveRemote; + let calls = 0; + const remote = new Promise((resolve) => { + resolveRemote = resolve; + }); + const service = new PortalService({ + db: createVersionDb(), + objectStorage: {}, + managementClient: { + isConfigured() { + return true; + }, + async call() { + calls += 1; + return await remote; + } + }, + authService: {} + }); + + const first = service.refreshVersionInfoCache({ timeoutMs: 1000 }); + const second = service.refreshVersionInfoCache({ timeoutMs: 2000 }); + expect(calls).toBe(1); + + resolveRemote({ + version: 1, + items: [{ group: "runtime", name: "core", packageName: "openclaw", version: "1.2.3" }] + }); + const [firstResult, secondResult] = await Promise.all([first, second]); + expect(firstResult.items).toEqual(secondResult.items); + expect(firstResult.items.map((item) => item.package_name)).toEqual(["openclaw"]); +}); + it("updates portal settings without touching uploaded logo data", () => { const calls = []; let settings = { diff --git a/packages/daedalus-agent-management-web/server/test/runtime-settings-route.test.js b/packages/daedalus-agent-management-web/server/test/runtime-settings-route.test.js index aac8b4b..6fee245 100644 --- a/packages/daedalus-agent-management-web/server/test/runtime-settings-route.test.js +++ b/packages/daedalus-agent-management-web/server/test/runtime-settings-route.test.js @@ -139,6 +139,71 @@ it("includes the env allowlist config in bootstrap", async () => { } }); +it("waits for the version cache before returning bootstrap", async () => { + let resolveVersionRefresh; + let versionRefreshCalled = false; + let versionInfo = { items: [{ name: "local", version: "1.0.0" }] }; + const versionRefresh = new Promise((resolve) => { + resolveVersionRefresh = () => { + versionInfo = { + items: [ + { name: "local", version: "1.0.0" }, + { name: "runtime", version: "2.0.0" } + ] + }; + resolve(versionInfo); + }; + }); + const app = express(); + app.use(express.json()); + app.use("/api", createRoutes(createBaseServices({ + portalService: { + getSettings() { + return {}; + }, + listAccessTokens() { + return []; + }, + getVersionInfo() { + return versionInfo; + }, + refreshVersionInfoCache() { + versionRefreshCalled = true; + return versionRefresh; + } + } + }))); + + const server = await listen(app); + try { + const address = server.address(); + const responsePromise = requestJson( + `http://127.0.0.1:${address.port}/api/bootstrap`, + { method: "GET", body: undefined } + ); + + await new Promise((resolve) => setTimeout(resolve, 25)); + expect(versionRefreshCalled).toBeTrue(); + + let responseSettled = false; + responsePromise.then(() => { + responseSettled = true; + }); + await new Promise((resolve) => setTimeout(resolve, 25)); + expect(responseSettled).toBeFalse(); + + resolveVersionRefresh(); + const response = await responsePromise; + expect(response.status).toBe(200); + expect(response.body.version_info.items).toEqual([ + { name: "local", version: "1.0.0" }, + { name: "runtime", version: "2.0.0" } + ]); + } finally { + await close(server); + } +}); + it("updates the env allowlist config and writes an audit log", async () => { const auditWrites = []; const updates = []; diff --git a/packages/daedalus-agent-management-web/src/app/App.jsx b/packages/daedalus-agent-management-web/src/app/App.jsx index 86f94e3..9b2f38f 100644 --- a/packages/daedalus-agent-management-web/src/app/App.jsx +++ b/packages/daedalus-agent-management-web/src/app/App.jsx @@ -181,7 +181,6 @@ export default function App() { const skillSync = boot?.skill_sync; const templateSync = boot?.template_sync; const systemSync = boot?.system_sync; - const usageRefresh = boot?.usage_refresh; const versionInfo = boot?.version_info; const envConfig = boot?.env_config || {}; const currentUserId = currentUser?.id; @@ -386,7 +385,6 @@ export default function App() { skillSyncStatus={skillSync} templateSyncStatus={templateSync} systemSyncStatus={systemSync} - usageRefreshStatus={usageRefresh} versionInfo={versionInfo} envConfig={envConfig} runtimeHealth={runtimeHealth} @@ -447,8 +445,7 @@ export default function App() { llm_sync: next.llm_sync, skill_sync: next.skill_sync, template_sync: next.template_sync, - system_sync: next.system_sync, - usage_refresh: next.usage_refresh + system_sync: next.system_sync })); return next; }} @@ -494,7 +491,7 @@ export default function App() { default: return null; } - }, [accessTokens, agentCreateTemplateName, agentSync, agentsRefreshKey, currentUser, dashboardRefreshKey, envConfig, llmSync, messageApi, refreshRuntimeHealth, runtimeHealth, runtimeHealthLoading, selectedKey, settings, skillSync, systemSync, templateSync, usageRefresh, versionInfo]); + }, [accessTokens, agentCreateTemplateName, agentSync, agentsRefreshKey, currentUser, dashboardRefreshKey, envConfig, llmSync, messageApi, refreshRuntimeHealth, runtimeHealth, runtimeHealthLoading, selectedKey, settings, skillSync, systemSync, templateSync, versionInfo]); return ( !isEmptyValue(summary[key])) - .map(({ key, label }) => `${label} ${summary[key]}`); - - return items.length > 0 ? items.join("，") : "-"; -} - function renderTableValue(value) { if (isEmptyValue(value)) { return "-"; @@ -156,51 +144,6 @@ function renderMergedSummary(syncItems) { return items.length > 0 ? items.join("；") : "-"; } -function buildSyncStatusRows({ action, taskStatus, summaryFields }) { - return [ - { - key: "action", - title: "同步命令", - value: action - }, - { - key: "status", - title: "当前状态", - value: renderTaskStatus(taskStatus) - }, - { - key: "triggerSource", - title: "触发来源", - value: taskStatus?.trigger_source || "-" - }, - { - key: "startedAt", - title: "开始时间", - value: taskStatus?.started_at || "-" - }, - { - key: "finishedAt", - title: "结束时间", - value: taskStatus?.finished_at || "-" - }, - { - key: "lastSuccessAt", - title: "上次成功时间", - value: taskStatus?.last_success_at || "-" - }, - { - key: "summary", - title: "同步摘要", - value: renderSummary(taskStatus?.summary, summaryFields) - }, - { - key: "error", - title: "最近错误", - value: taskStatus?.error_message || "-" - } - ]; -} - function formatJson(value) { if (value === undefined || value === null) { return "-"; @@ -377,17 +320,6 @@ function buildCatalogSyncRows({ ]; } -function buildUsageSyncRows(usageRefreshStatus) { - return buildSyncStatusRows({ - action: "model-gateway.usage.snapshot", - taskStatus: usageRefreshStatus, - summaryFields: [ - { key: "refreshed_agents", label: "刷新数字员工" }, - { key: "captured_at", label: "采集时间" } - ] - }); -} - function buildVersionRows(versionInfo) { return Array.isArray(versionInfo?.items) ? versionInfo.items.map((item, index) => ({ @@ -524,7 +456,6 @@ export function SettingsPage({ skillSyncStatus, templateSyncStatus, systemSyncStatus, - usageRefreshStatus, versionInfo, envConfig, runtimeHealth, @@ -750,30 +681,27 @@ export function SettingsPage({ const handleCatalogSync = async () => { setCatalogSyncing(true); try { - const next = await runWithOperationMask("正在同步数据，请稍候...", async () => ( + const next = await runWithOperationMask("正在同步目录，请稍候...", async () => ( await onCatalogSync() )); const catalogStatus = next?.result?.catalog?.status; - const usageStatus = next?.result?.usage?.status; - if (catalogStatus === "success" && usageStatus === "success") { - notify?.success("数据同步完成"); + if (catalogStatus === "success") { + notify?.success("目录同步完成"); return; } - if (catalogStatus === "skipped" || usageStatus === "skipped") { + if (catalogStatus === "skipped") { notify?.warning( next?.result?.catalog?.error_message - || next?.result?.usage?.error_message - || "数据同步已跳过" + || "目录同步已跳过" ); return; } notify?.error( next?.result?.catalog?.error_message - || next?.result?.usage?.error_message - || "数据同步失败" + || "目录同步失败" ); } catch (error) { - notify?.error(`数据同步失败：${getErrorMessage(error, "同步失败")}`); + notify?.error(`目录同步失败：${getErrorMessage(error, "同步失败")}`); } finally { setCatalogSyncing(false); } @@ -874,7 +802,6 @@ export function SettingsPage({ render: renderTableValue } ]; - const usageSyncRows = buildUsageSyncRows(usageRefreshStatus); const versionRows = buildVersionRows(versionInfo); const runtimeCredentialRows = buildRuntimeCredentialRows(envConfig, runtimeHealth); const runtimeCredentialColumns = [ @@ -1232,7 +1159,7 @@ export function SettingsPage({ void handleCatalogSync(); }} > - 同步数据 + 同步目录
- diff --git a/packages/daedalus-agent-management-web/src/styles.css b/packages/daedalus-agent-management-web/src/styles.css index 841b20e..c8e4075 100644 --- a/packages/daedalus-agent-management-web/src/styles.css +++ b/packages/daedalus-agent-management-web/src/styles.css @@ -1588,11 +1588,11 @@ body { border-bottom: 1px solid rgba(15, 23, 42, 0.08); } -.settings-catalog-sync-table { - margin-top: 0; +.settings-sync-section-wide { + grid-column: 1 / -1; } -.settings-token-sync-table { +.settings-catalog-sync-table { margin-top: 0; } diff --git a/packages/daedalus-agent-management-web/src/utils/startupSyncGate.js b/packages/daedalus-agent-management-web/src/utils/startupSyncGate.js index f7ba434..b1f3f02 100644 --- a/packages/daedalus-agent-management-web/src/utils/startupSyncGate.js +++ b/packages/daedalus-agent-management-web/src/utils/startupSyncGate.js @@ -11,7 +11,6 @@ const INITIAL_SYNC_PENDING_STATUSES = new Set(["idle", "running"]); const SYNC_STATUS_KEYS = [ "agent_sync", "llm_sync", - "usage_refresh", "skill_sync", "template_sync", "system_sync" diff --git a/packages/daedalus-agent-management-web/src/utils/startupSyncGate.test.js b/packages/daedalus-agent-management-web/src/utils/startupSyncGate.test.js index 6f515b2..6322577 100644 --- a/packages/daedalus-agent-management-web/src/utils/startupSyncGate.test.js +++ b/packages/daedalus-agent-management-web/src/utils/startupSyncGate.test.js @@ -37,7 +37,7 @@ it("returns page-specific pending dependencies", () => { }); it("polls while any task runs or the selected page is initially blocked", () => { - expect(shouldPollBootstrapForSync("dashboard", { usage_refresh: { status: "running" } })).toBe(true); + expect(shouldPollBootstrapForSync("dashboard", { agent_sync: { status: "running" } })).toBe(true); expect(shouldPollBootstrapForSync("templates", { template_sync: { status: "idle" } })).toBe(true); expect(shouldPollBootstrapForSync("templates", { template_sync: { status: "success" } })).toBe(false); }); diff --git a/packages/daedalus-agent-management-web/test/run-browser-e2e.js b/packages/daedalus-agent-management-web/test/run-browser-e2e.js index 6a91cda..a86ad0b 100644 --- a/packages/daedalus-agent-management-web/test/run-browser-e2e.js +++ b/packages/daedalus-agent-management-web/test/run-browser-e2e.js @@ -745,7 +745,6 @@ async function main() { skill_sync: { status: "success", last_success_at: "2026-05-26T00:00:00.000Z", summary: {} }, template_sync: { status: "success", last_success_at: "2026-05-26T00:00:00.000Z", summary: {} }, system_sync: { status: "success", last_success_at: "2026-05-26T00:00:00.000Z", summary: {} }, - usage_refresh: { status: "success", last_success_at: "2026-05-26T00:00:00.000Z", summary: {} }, access_tokens: accessTokens, version_info: { items: [ @@ -1477,8 +1476,7 @@ async function main() { skill_sync: { status: "success", summary: { skills: 2 } }, template_sync: { status: "success", summary: { templates: 2 } }, system_sync: { status: "success", summary: { systems: 2 } }, - usage_refresh: { status: "success", summary: { refreshed_agents: 1 } }, - result: { catalog: { status: "success" }, usage: { status: "success" } } + result: { catalog: { status: "success" } } })); return; } diff --git a/packages/daedalus-agent-model-gateway/src/server.js b/packages/daedalus-agent-model-gateway/src/server.js index 9d1316d..e3dccb2 100644 --- a/packages/daedalus-agent-model-gateway/src/server.js +++ b/packages/daedalus-agent-model-gateway/src/server.js @@ -85,6 +85,8 @@ export function createGatewayServer({ store, runtimeToken = "", port = 3040, hos const timeout = setTimeout(() => controller.abort(), upstreamTimeoutMs); const startedAt = Date.now(); const requestId = String(req.headers["x-request-id"] || randomUUID()); + // Runtime Token 供 OpenClaw 使用；MQTT 插件会补充带 senderId 的可计费用量事件。 + const billable = auth.agentId !== "runtime"; const upstreamBody = { ...body, model: resolved.modelId }; try { const upstream = await fetch(endpoint, { method: "POST", headers: { Authorization: `Bearer ${apiKey}`, "Content-Type": "application/json", Accept: body.stream ? "text/event-stream" : "application/json" }, body: JSON.stringify(upstreamBody), signal: controller.signal, redirect: "error" }); @@ -115,7 +117,7 @@ export function createGatewayServer({ store, runtimeToken = "", port = 3040, hos } } res.end(); - await recordUsage({ eventId: requestId, requestId, agentId: auth.agentId || "runtime", senderId: "unknown", requestedModelRef: requestedModel, actualProviderId: resolved.providerId, actualModelRef: `${resolved.providerId}/${resolved.modelId}`, ...usage, totalTokens: usage.totalTokens || usage.promptTokens + usage.completionTokens, latencyMs: Date.now() - startedAt }); + await recordUsage({ eventId: requestId, requestId, agentId: auth.agentId || "runtime", senderId: "unknown", requestedModelRef: requestedModel, actualProviderId: resolved.providerId, actualModelRef: `${resolved.providerId}/${resolved.modelId}`, ...usage, totalTokens: usage.totalTokens || usage.promptTokens + usage.completionTokens, billable, latencyMs: Date.now() - startedAt }); return; } const text = await upstream.text(); @@ -124,7 +126,7 @@ export function createGatewayServer({ store, runtimeToken = "", port = 3040, hos res.writeHead(upstream.status, { "Content-Type": upstream.headers.get("content-type") || "application/json" }); res.end(JSON.stringify(parsed)); const usage = usageFromBody(parsed); - await recordUsage({ eventId: requestId, requestId, agentId: auth.agentId || "runtime", senderId: "unknown", requestedModelRef: requestedModel, actualProviderId: resolved.providerId, actualModelRef: `${resolved.providerId}/${resolved.modelId}`, ...usage, totalTokens: usage.totalTokens || usage.promptTokens + usage.completionTokens, latencyMs: Date.now() - startedAt }); + await recordUsage({ eventId: requestId, requestId, agentId: auth.agentId || "runtime", senderId: "unknown", requestedModelRef: requestedModel, actualProviderId: resolved.providerId, actualModelRef: `${resolved.providerId}/${resolved.modelId}`, ...usage, totalTokens: usage.totalTokens || usage.promptTokens + usage.completionTokens, billable, latencyMs: Date.now() - startedAt }); } catch (error) { const timeoutError = error?.name === "AbortError"; if (!res.headersSent) json(res, 504, { error: { code: timeoutError ? "upstream_timeout" : "upstream_unavailable", message: timeoutError ? "upstream request timed out" : "upstream provider unavailable" } }); diff --git a/packages/daedalus-agent-model-gateway/test/server.test.js b/packages/daedalus-agent-model-gateway/test/server.test.js index bbfcd94..22745a2 100644 --- a/packages/daedalus-agent-model-gateway/test/server.test.js +++ b/packages/daedalus-agent-model-gateway/test/server.test.js @@ -43,6 +43,17 @@ test("requires gateway token and proxies OpenAI-compatible chat requests", async } assert.equal(usageEvents.length, 1); assert.equal(usageEvents[0].actualModelRef, "gcapi/deepseek-v4-flash"); + assert.equal(usageEvents[0].billable, false); + const client = store.issueClient({ agentId: "finance-agent" }); + const clientResponse = await fetch(`http://127.0.0.1:${port}/v1/chat/completions`, { method: "POST", headers: { Authorization: `Bearer ${client.token}`, "content-type": "application/json" }, body: JSON.stringify({ model: "gcapi/deepseek-v4-flash", messages: [{ role: "user", content: "hi" }] }) }); + assert.equal(clientResponse.status, 200); + await clientResponse.arrayBuffer(); + for (let attempt = 0; attempt < 20 && usageEvents.length < 2; attempt += 1) { + await new Promise((resolve) => setTimeout(resolve, 10)); + } + assert.equal(usageEvents.length, 2); + assert.equal(usageEvents[1].agentId, "finance-agent"); + assert.equal(usageEvents[1].billable, true); await gateway.close(); await new Promise((resolve) => upstream.close(resolve)); await new Promise((resolve) => usage.close(resolve)); diff --git a/test/case/integration/playwright-flow.js b/test/case/integration/playwright-flow.js index 1f39ed7..51e2379 100644 --- a/test/case/integration/playwright-flow.js +++ b/test/case/integration/playwright-flow.js @@ -1422,7 +1422,11 @@ async function useMcpFromAgent(page, baseUrl, timeoutMs) { page, baseUrl, AGENT_ID, - `MCP_USAGE_CHECK ${AGENT_ID}`, + [ + `MCP_USAGE_CHECK ${AGENT_ID}`, + `请直接调用 MCP 工具 integration-mcp__employee_template_lookup，参数 employeeId=${AGENT_ID}。`, + "不要调用 exec、process 或其它工具；调用完成后再回复调用结果。" + ].join("\n"), timeoutMs, exactReplyExpected ? { expectedText, maxReplyLength: 1200 } -- Gitee