From 06c4549aa0c87a1d658bbdb2403aadc7a47cde02 Mon Sep 17 00:00:00 2001 From: EMRG Evolution Date: Wed, 2 Sep 2026 15:43:36 +0800 Subject: [PATCH] emrg: GUI image input via clipboard paste (Cmd+V) and file drag-drop MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Render chain for GUI image attachments, aligned with the TUI's [📷 label] placeholder semantics so both ends interoperate in the same session: - Composer (tiptap): handlePaste detects clipboard image/* files (inserts accompanying text/plain first, TUI behavior); handleDrop accepts dragged image files; each image is base64-sent to the new emrg:saveImage IPC, then a literal [📷 label] placeholder is inserted at the cursor and the attachment recorded in pending state. - main.js: new emrg:saveImage handler writes into the active session's images directory ({safeLabel}_{hash8}.{ext}, TUI naming convention, extension allowlist png/jpg/gif/webp/bmp/svg, dedup by existence); sendMessage now validates and forwards an images array to sendTask (daemon already supported it). - preload.js exposes saveImage (invoke API 53 -> 54, guard test updated). - composer.ts helpers (pure, unit-tested): imagePlaceholder/toSafeImageLabel mirror TUI safe_label; normalizePlaceholders reverts tiptap-markdown's backslash-escaped [📷 ...]; resolveSendImages keeps only attachments whose placeholder is still in the text and computes character positions (deleting the placeholder drops the image, TUI semantics). Images survive the busy queue/requeue paths in both Composer and daemonBridge. - daemon.py: _build_user_content now uses images[].mime for the data URL instead of hardcoding image/png (GUI drag-in JPEG sent wrong mime); defaults to png for TUI/history records. 2 new daemon tests. Verification: pytest 1224 passed/1 skipped (1225 collected), renderer typecheck clean + 508 tests/45 files, GUI node suite 93 pass/0 fail, doc-count guard 5/5, import + CLI green. Agent.md counts synced. --- Agent.md | 4 +- emrg/gui/main.js | 57 ++++- emrg/gui/preload.js | 2 + emrg/gui/renderer/src/components/Composer.tsx | 195 +++++++++++++++++- emrg/gui/renderer/src/lib/composer.test.ts | 68 ++++++ emrg/gui/renderer/src/lib/composer.ts | 64 ++++++ emrg/gui/renderer/src/lib/daemonBridge.ts | 6 +- emrg/gui/test/preload-api.test.js | 9 +- emrg/server/daemon.py | 7 +- tests/test_daemon.py | 45 ++++ 10 files changed, 440 insertions(+), 17 deletions(-) diff --git a/Agent.md b/Agent.md index 6730d607..d036ae4e 100644 --- a/Agent.md +++ b/Agent.md @@ -119,9 +119,9 @@ Community needs voiced in HN agent-UI discussions map directly to EMRG's design: pkill -f "emrg.server"; rm -f ~/.emrg/emrgd.token; python -m emrg ``` -Python: `uv run pytest tests/ -v` (1223) — import check: `uv run python -c "from emrg.client.app import run_client"` +Python: `uv run pytest tests/ -v` (1225) — import check: `uv run python -c "from emrg.client.app import run_client"` GUI: `cd emrg/gui && npm test` (100: 44 daemon_client + 20 conn-manager + 7 integration + 7 nav-policy + 7 gui-state + 6 build-config + 4 boot-contract + 3 preload-api + 2 theme-guard) — syntax: `node --check main.js preload.js daemon_client.js` -Renderer: `cd emrg/gui/renderer && npm run typecheck && npm test` (499: 5 snapshot-store + 9 utils + 3 ErrorBoundary + 2 App smoke + 11 commands + 4 copywriting + 11 i18n + 11 markdown + 21 transcript + 10 TranscriptView + 15 history + 22 composer + 38 Composer + 6 LinkDialog + 16 sidebar + 17 Sidebar + 9 fileTree + 9 FileTree + 16 resultPanel + 8 ResultPanel + 27 workspaceView + 10 WorkspaceView + 10 dialog + 6 Dialog + 9 ConfirmDialog + 9 RenameDialog + 10 dialogLists + 3 HelpDialog + 9 MemoryDialog + 6 SkillsDialog + 8 openSession + 6 WelcomeDialog + 9 OpenSessionDialog + 7 NewSessionDialog + 7 rewind + 8 RewindDialog + 7 GithubDeviceDialog + 18 daemonBridge + 7 DaemonBridgeProvider + 30 Shell + 15 DialogHost + 20 SettingsPanel + 6 TaskFormDialog + 5 RantDialog + 4 vendorMarkdown) + `npm run build` → `renderer/dist/` +Renderer: `cd emrg/gui/renderer && npm run typecheck && npm test` (508: 5 snapshot-store + 9 utils + 3 ErrorBoundary + 2 App smoke + 11 commands + 4 copywriting + 11 i18n + 11 markdown + 21 transcript + 10 TranscriptView + 15 history + 31 composer + 38 Composer + 6 LinkDialog + 16 sidebar + 17 Sidebar + 9 fileTree + 9 FileTree + 16 resultPanel + 8 ResultPanel + 27 workspaceView + 10 WorkspaceView + 10 dialog + 6 Dialog + 9 ConfirmDialog + 9 RenameDialog + 10 dialogLists + 3 HelpDialog + 9 MemoryDialog + 6 SkillsDialog + 8 openSession + 6 WelcomeDialog + 9 OpenSessionDialog + 7 NewSessionDialog + 7 rewind + 8 RewindDialog + 7 GithubDeviceDialog + 18 daemonBridge + 7 DaemonBridgeProvider + 30 Shell + 15 DialogHost + 20 SettingsPanel + 6 TaskFormDialog + 5 RantDialog + 4 vendorMarkdown) + `npm run build` → `renderer/dist/` CI: `uv run pytest` (ubuntu + **windows-2025 matrix** — Windows pytest 回归在 PR CI 即失败，v0.2.29 教训 #725) + GUI tests + **actionlint workflow lint** (`rhysd/actionlint@v1.7.12` gate, #444 — workflow 解析错误在 PR CI 即失败，如 `if:` secrets 上下文) Re-trigger: `scripts/re-trigger-ci.sh [branch]` (workflow_dispatch, #527 — 替代空 commit 重触发：Actions outage 会整段丢弃 push 事件，dispatch 走 API 路径不受影响) Git-over-https 兜底: `python scripts/sync-master-from-api.py [--repo owner/name] [--ref master]` — 受限网络下 github.com:443 不可达而 api.github.com 可达时，用 Git Data API 的 verification payload + signature 字节级重建上游 commit（含 web-flow GPG 签名 squash merge，reconstruct_commit 经 hermetic 测试验证 sha 一致）并推进本地 refs；内容对象缺失时 fail-loud 提示改用 git fetch（10+ 周期实证的恢复路径） diff --git a/emrg/gui/main.js b/emrg/gui/main.js index 584374b5..6ef9c428 100644 --- a/emrg/gui/main.js +++ b/emrg/gui/main.js @@ -11,6 +11,7 @@ const os = require("os"); const path = require("path"); const { pathToFileURL } = require("url"); const { spawn } = require("child_process"); +const crypto = require("crypto"); const { parse: parseToml, stringify: stringifyToml } = require("smol-toml"); const { generateSessionId, SESSION_ID_RE } = require("./daemon_client"); const { ConnManager } = require("./conn-manager"); @@ -279,6 +280,25 @@ vision = false return typeof t === "string" && t.length > 0 && t.length <= 20000; } + // rant 2026-09-02T15:23:53：images 透传护栏（{path,label?,position?,mime?}[]） + function validateImages(imgs) { + if (imgs === null || imgs === undefined) return true; + if (!Array.isArray(imgs) || imgs.length > 12) return false; + for (const it of imgs) { + if (!it || typeof it !== "object") return false; + if (typeof it.path !== "string" || !it.path || it.path.length > 1024) return false; + if (it.label !== undefined && (typeof it.label !== "string" || it.label.length > 200)) return false; + if ( + it.position !== undefined && + (typeof it.position !== "number" || !Number.isInteger(it.position) || it.position < 0 || it.position > 20000) + ) { + return false; + } + if (it.mime !== undefined && (typeof it.mime !== "string" || !/^image\/[\w.+-]+$/.test(it.mime))) return false; + } + return true; + } + function validateConfig(c) { // 设计 §7.1：直接接收所需字段 + 基本类型检查（防写坏 config.toml 的健壮性，非安全设计） const out = {}; @@ -336,9 +356,10 @@ vision = false }; }); - ipcMain.handle("emrg:sendMessage", async (_e, { sessionId, text, requestId, sandbox }) => { + ipcMain.handle("emrg:sendMessage", async (_e, { sessionId, text, requestId, sandbox, images }) => { if (!validateSessionId(sessionId)) throw new Error("invalid session_id"); if (!validateText(text)) throw new Error("invalid text"); + if (!validateImages(images)) throw new Error("invalid images"); // rant 2026-09-02T15:23:53 if (requestId !== undefined && (typeof requestId !== "string" || requestId.length < 8 || requestId.length > 64)) { throw new Error("invalid request_id"); // G143：renderer 预生成 id 的格式护栏 } @@ -355,7 +376,8 @@ vision = false // G143：renderer 预生成 requestId（send 前标记自有流，消除 IPC 往返竞态窗口） // Rant 2026-08-20T18:18：sandbox 档位（read-only / workspace-write / danger-full-access） // G65：conn.sendTask 内部标记 ownStream（每连接独立锁） - rid = conn.sendTask({ sessionId, cwd: sessionCwd, prompt: text, requestId, sandbox }); + // rant 2026-09-02T15:23:53：images 透传（daemon_client.sendTask 早已支持） + rid = conn.sendTask({ sessionId, cwd: sessionCwd, prompt: text, requestId, sandbox, images: images ?? null }); } catch (e) { conn._releaseOwnStream(); // sendTask 抛异常（ws.send 失败）→ 释放锁，防 G65 锁泄漏 throw e; @@ -363,6 +385,37 @@ vision = false return { ok: true, requestId: rid }; // G124：回传 requestId → renderer 识别自有流 }); + // rant 2026-09-02T15:23:53：图片落盘——/.emrg/sessions//images/ + // {safeLabel}_{blake2b4}.{ext}（TUI 同款命名约定：label 清洗 + 8-hex hash 前缀， + // 同图重复粘贴去重）。⚠️ 哈希算法与 TUI 不同：TUI 用 Python blake2b(digest_size=4) + // （Node crypto 不支持变长 blake2b），GUI 取 blake2b-512 前 8 hex —— 两端同字节 + // 文件名 hash 不同（同一图在 TUI/GUI 各贴一次会各存一份，功能无碍，仅存储）， + // GUI 自身去重自洽（同字节 → 同文件名 → 跳过写盘）。 + ipcMain.handle("emrg:saveImage", async (_e, { sessionId, data, label, mime } = {}) => { + if (!validateSessionId(sessionId)) throw new Error("invalid session_id"); + if (typeof data !== "string" || !data || data.length > 28_000_000) throw new Error("invalid image data"); // ~21MB base64 + if (typeof label !== "string" || !label || label.length > 200) throw new Error("invalid label"); + const mm = /^image\/([\w.+-]+)$/.exec(mime || "image/png"); + if (!mm) throw new Error("invalid mime"); + // 扩展名白名单（与 Composer SUPPORTED_IMAGE_MIME 一致） + const EXT_BY_SUB = { png: "png", jpeg: "jpg", gif: "gif", webp: "webp", bmp: "bmp", "svg+xml": "svg" }; + const ext = EXT_BY_SUB[mm[1].toLowerCase()]; + if (!ext) throw new Error("unsupported image type"); + const buf = Buffer.from(data, "base64"); + if (buf.length === 0) throw new Error("invalid image data"); + const projectRoot = resolveSessionCwd(sessionId) || DEFAULT_CWD; + const imagesDir = path.join(projectRoot, ".emrg", "sessions", sessionId, "images"); + fs.mkdirSync(imagesDir, { recursive: true }); + // 文件名字面清洗镜像 TUI safe_label（. 保留以便 .png 类后缀友好，去尾部 ._） + const safeLabel = label.replace(/[^a-zA-Z0-9._-]/g, "_").slice(0, 40).replace(/[._]+$/, "") || "image"; + const h = crypto.createHash("blake2b512").update(buf).digest("hex").slice(0, 8); + const filename = `${safeLabel}_${h}.${ext}`; + const finalPath = path.join(imagesDir, filename); + if (!fs.existsSync(finalPath)) fs.writeFileSync(finalPath, buf); // 同名去重 + logger.info(`[gui:saveImage] ${filename} (${buf.length} bytes)`); + return { path: finalPath, mime: `image/${mm[1].toLowerCase()}` }; + }); + ipcMain.handle("emrg:listSessions", async () => listSessions()); // Rant 2026-08-21T12:44:34：一键"重启生效"。旧实现只发 shutdown —— daemon 会被 diff --git a/emrg/gui/preload.js b/emrg/gui/preload.js index 454699e0..1879e3f7 100644 --- a/emrg/gui/preload.js +++ b/emrg/gui/preload.js @@ -9,6 +9,8 @@ const { contextBridge, ipcRenderer } = require("electron"); const api = { init: () => ipcRenderer.invoke("emrg:init"), sendMessage: (payload) => ipcRenderer.invoke("emrg:sendMessage", payload), + // rant 2026-09-02T15:23:53：图片落盘（粘贴/拖拽 → base64 → main 写 /.emrg/sessions//images/） + saveImage: (payload) => ipcRenderer.invoke("emrg:saveImage", payload), listSessions: () => ipcRenderer.invoke("emrg:listSessions"), restartDaemon: () => ipcRenderer.invoke("emrg:restartDaemon"), relaunchGui: () => ipcRenderer.invoke("emrg:relaunchGui"), diff --git a/emrg/gui/renderer/src/components/Composer.tsx b/emrg/gui/renderer/src/components/Composer.tsx index a39d37cc..d38e9263 100644 --- a/emrg/gui/renderer/src/components/Composer.tsx +++ b/emrg/gui/renderer/src/components/Composer.tsx @@ -10,7 +10,12 @@ import { menuForPrefix, menuNavigate, queueSend, + imagePlaceholder, + toSafeImageLabel, + normalizePlaceholders, + resolveSendImages, type CmdMenuState, + type ImageAttach, } from "../lib/composer"; import { genRequestId } from "../lib/utils"; import { useI18n } from "../lib/i18n"; @@ -40,10 +45,24 @@ export interface SendOptions { text: string; requestId: string; sandbox?: string | null; + /** 图片附件（rant 2026-09-02T15:23:53：粘贴/拖拽 → 落盘 → 随消息发送） */ + images?: ImageAttach[] | null; } export interface SendResult { requestId?: string; } +/** emrg:saveImage 入参（renderer → main IPC，base64 数据 + 展示标签） */ +export interface SaveImagePayload { + sessionId?: string | null; + data: string; + label: string; + mime?: string; +} +/** emrg:saveImage 返回值（落盘路径 + 归一 mime） */ +export interface SaveImageResult { + path: string; + mime?: string; +} /** 指令路由入参（parseInput 结果；type:"unknown" 无 args——vanilla 走 default 提示） */ export interface CommandRouting { type: "command" | "unknown"; @@ -59,6 +78,8 @@ export interface ComposerProps { busy?: boolean; /** 注入发送函数（默认 window.emrg.sendMessage；测试传假实现） */ sendMessage?: (opts: SendOptions) => Promise; + /** 注入图片落盘函数（默认 window.emrg.saveImage；测试传假实现） */ + saveImage?: (payload: SaveImagePayload) => Promise; /** / 指令路由回调（Batch 5 接线：/clear /model /memory …） */ onCommand?: (routing: CommandRouting) => void; /** 测试注入：挂载后回填 tiptap Editor 实例（命令驱动测试用） */ @@ -110,6 +131,7 @@ export function Composer({ sandbox = null, busy: busyProp, sendMessage: send, + saveImage: saveImageProp, onCommand, editorRef, }: ComposerProps) { @@ -124,7 +146,13 @@ export function Composer({ // ⚠️ (rant 2026-08-28T22:27:01) 链接改用应用内对话框收集 URL（Electron 禁用 window.prompt） const [linkDialogOpen, setLinkDialogOpen] = useState(false); const [linkHref, setLinkHref] = useState(""); - const queuedRef = useRef>>(new Map()); + const queuedRef = useRef>>(new Map()); + // 图片附加（rant 2026-09-02T15:23:53）：pending 语义对齐 TUI _pending_images—— + // 占位符删除即弃图（发送时 filter）；ref 镜像供 tiptap 一次性闭包读写 + const [pending, setPending] = useState([]); + const pendingRef = useRef([]); + const attachRef = useRef<(files: File[], at?: number | null) => void>(() => {}); + const insertRawRef = useRef<(text: string, at?: number | null) => void>(() => {}); // tiptap 选项闭包只创建一次 → 变化值走 ref 桥接（menu/submit/selectCmd/t） const menuRef = useRef(menu); menuRef.current = menu; @@ -157,6 +185,17 @@ export function Composer({ Promise.reject(new Error("window.emrg.sendMessage unavailable")); }); + // 图片落盘函数解析（rant 2026-09-02T15:23:53：默认走 preload 桥 emrg:saveImage） + const saveFn = + saveImageProp ?? + ((payload: SaveImagePayload) => + (window as unknown as { emrg?: { saveImage?: (p: SaveImagePayload) => Promise } }).emrg + ?.saveImage?.(payload) ?? Promise.reject(new Error("window.emrg.saveImage unavailable"))); + const saveRef = useRef(saveFn); + saveRef.current = saveFn; + // tiptap 一次性闭包内读写 editor 的桥（editor 变量在 useEditor 之后才声明） + const editorBoxRef = useRef(null); + const editor = useEditor({ extensions: [ StarterKit.configure({ link: { openOnClick: false } }), @@ -226,6 +265,38 @@ export function Composer({ // Stage 2 粘贴安全：HTML 粘贴路径经 DOMPurify 净化（script/事件属性剥离； // tiptap-markdown 的纯文本粘贴分支本身无 HTML 注入面） transformPastedHTML: (html) => DOMPurify.sanitize(html), + // rant 2026-09-02T15:23:53：剪贴板图片粘贴——同带 text/plain 先插文本 + // （对齐 TUI「文本粘贴后追加图」），图片逐张走「附加图片」落盘 + 占位符 + handlePaste: (_view, event) => { + const cd = event.clipboardData; + if (!cd) return false; + const files = Array.from(cd.files || []); + const imgs = files.filter((f) => f.type && f.type.startsWith("image/")); + if (imgs.length === 0) return false; + event.preventDefault(); + const text = cd.getData("text/plain"); + const ed = editorBoxRef.current; + if (text && ed) { + const from = ed.state.selection.from; + insertRawRef.current(text, from); + attachRef.current(imgs, from + text.length); + } else { + attachRef.current(imgs, null); + } + return true; + }, + // rant 2026-09-02T15:23:53：拖拽图片文件进输入区 + handleDrop: (_view, event) => { + const dt = event.dataTransfer; + if (!dt) return false; + const files = Array.from(dt.files || []); + if (files.length === 0) return false; + const imgs = files.filter((f) => f.type && f.type.startsWith("image/")); + if (imgs.length === 0) return false; + event.preventDefault(); + attachRef.current(imgs, null); + return true; + }, }, onUpdate: ({ editor }) => { const text = editor.getText(); @@ -269,12 +340,17 @@ export function Composer({ setHasText(false); } prevSidRef.current = sid; + // 会话切换：pending 图片属旧会话输入（TUI 语义——占位符即图的生命周期），丢弃 + pendingRef.current = []; + setPending([]); }, [sid, editor, store]); - // 测试注入：editor 实例回填 + // 测试注入：editor 实例回填（兼作 tiptap 一次性闭包的实例桥） useEffect(() => { + editorBoxRef.current = editor; if (editorRef) editorRef.current = editor; return () => { + editorBoxRef.current = null; if (editorRef) editorRef.current = null; }; }, [editor, editorRef]); @@ -334,9 +410,15 @@ export function Composer({ async function submit(): Promise { if (!editor) return; // tiptap-markdown 的 storage 类型不并入 Editor.storage（Storage 泛型）——运行时存在，显式取值 - const md = (editor.storage as unknown as { markdown: { getMarkdown(): string } }).markdown.getMarkdown(); + const mdRaw = (editor.storage as unknown as { markdown: { getMarkdown(): string } }).markdown.getMarkdown(); + // rant 2026-09-02T15:23:53：tiptap-markdown 会把占位符转义成 \[📷 …\]（防被当链接语法）， + // 发送前还原字面量——daemon 收到的文本与 TUI 完全一致（历史互操作） + const md = normalizePlaceholders(mdRaw, pendingRef.current.map((p) => p.label)); const value = md.trim(); if (!value) return; + // 图片收敛（TUI 同语义）：占位符仍在文本中的保留（删了占位符即弃图）； + // position = 字面占位符在文本中的字符偏移（daemon 按 position 切块） + const images = resolveSendImages(pendingRef.current, value); const parsed = parseInput(value); if (parsed.type !== "message") { // 指令：清输入 + 关菜单 + 路由（vanilla rant 19:44 P1：/ 开头不进 sendMessage） @@ -364,11 +446,20 @@ export function Composer({ const requestId = genRequestId(); store.setOwnStream(requestId); if (wasBusy) { - queueSend(queuedRef.current, sid, { requestId, text: value, sandbox: tier }); + queueSend(queuedRef.current, sid, { requestId, text: value, sandbox: tier, images }); } try { - const res = await sendFn({ sessionId: sid, text: value, requestId, sandbox: tier }); + const res = await sendFn({ + sessionId: sid, + text: value, + requestId, + sandbox: tier, + images: images.length ? images : null, + }); store.setOwnStream(res.requestId || requestId); // G124：以 daemon 回显为准 + // 发送成功 → 清 pending（图片已随消息交给 daemon；失败保留以便重发） + pendingRef.current = []; + setPending([]); } catch { setInternalBusy(false); store.setOwnStream(null); @@ -390,8 +481,100 @@ export function Composer({ const selectCmdRef = useRef<(cmd: string) => void>(selectCmd); selectCmdRef.current = selectCmd; + /* ── 图片附加（rant 2026-09-02T15:23:53：粘贴/拖拽 → IPC 落盘 → 光标处占位符）── */ + + /** 原始文本直插（不经 tiptap-markdown 解析——粘贴文本/占位符都要字面量） */ + function insertRawText(text: string, at?: number | null): void { + const ed = editorBoxRef.current; + if (!ed) return; + const pos = at ?? ed.state.selection.from; + ed.view.dispatch(ed.state.tr.insertText(text, pos)); + } + insertRawRef.current = insertRawText; + + /** File → base64（FileReader，保留字节序与 data URL 解码一致） */ + function fileToBase64(file: File): Promise { + return new Promise((resolve, reject) => { + const r = new FileReader(); + r.onload = () => { + const dataUrl = String(r.result || ""); + resolve(dataUrl.split(",", 2)[1] ?? ""); + }; + r.onerror = () => reject(r.error ?? new Error("FileReader error")); + r.readAsDataURL(file); + }); + } + + /** 受支持图片 mime（与 main.js emrg:saveImage 的扩展名白名单一致） */ + const SUPPORTED_IMAGE_MIME: ReadonlySet = new Set([ + "image/png", + "image/jpeg", + "image/gif", + "image/webp", + "image/bmp", + "image/svg+xml", + ]); + + /** + * 逐张「附加图片」：落盘（IPC saveImage，返回绝对路径）→ 光标处插入占位符 → + * 记 pendingImages（path/label/position 语义同 TUI _pending_images）。 + */ + async function attachImages(files: File[], at?: number | null): Promise { + const ed = editorBoxRef.current; + const sid = sidRef.current; + if (!ed || !sid) return; + const imgs = files.filter((f) => f.type && SUPPORTED_IMAGE_MIME.has(f.type.toLowerCase())); + if (imgs.length === 0) return; + let pos = at ?? ed.state.selection.from; + for (const file of imgs) { + const mime = file.type.toLowerCase(); + let b64: string; + try { + b64 = await fileToBase64(file); + } catch { + continue; + } + const n = pendingRef.current.length + 1; + const stem = file.name ? file.name.replace(/\.[^.]+$/, "") : ""; + const display = stem ? toSafeImageLabel(stem) : `Image ${n}`; + let res: SaveImageResult; + try { + res = await saveRef.current({ sessionId: sid, data: b64, label: display, mime }); + } catch { + continue; // 落盘失败（无会话/类型不支持/超限）→ 跳过，不插幽灵占位符 + } + const placeholder = imagePlaceholder(display); + insertRawText(placeholder, pos); + pos += placeholder.length; + pendingRef.current = [...pendingRef.current, { path: res.path, label: placeholder, mime: res.mime || mime }]; + setPending(pendingRef.current); + } + ed.commands.focus(); + } + attachRef.current = (files, at) => { + void attachImages(files, at); + }; + return ( -
+
{ + // 允许图片文件拖入（否则浏览器显示 no-drop，drop 事件不触发） + const t = e.dataTransfer?.types; + if (t && Array.from(t).includes("Files")) e.preventDefault(); + }} + onDrop={(e) => { + if (e.defaultPrevented) return; // tiptap handleDrop 已处理（编辑器内） + const dt = e.dataTransfer; + if (!dt) return; + const files = Array.from(dt.files || []); + const imgs = files.filter((f) => f.type && f.type.startsWith("image/")); + if (!imgs.length) return; + e.preventDefault(); + attachRef.current(imgs, null); + }} + > {menu.items.length > 0 ? (
{menu.items.map((it, i) => ( diff --git a/emrg/gui/renderer/src/lib/composer.test.ts b/emrg/gui/renderer/src/lib/composer.test.ts index 3aa23782..7b16f2d5 100644 --- a/emrg/gui/renderer/src/lib/composer.test.ts +++ b/emrg/gui/renderer/src/lib/composer.test.ts @@ -2,11 +2,15 @@ import { describe, expect, it } from "vitest"; import { CMD_MENU_CLOSED, clearQueued, + imagePlaceholder, menuForPrefix, menuNavigate, + normalizePlaceholders, partitionRequeue, queueSend, + resolveSendImages, steerCommitted, + toSafeImageLabel, trackAfterResend, } from "./composer"; @@ -176,3 +180,67 @@ describe("clearQueued", () => { expect(clearQueued(new Map(), "s1")).toBe(false); }); }); + +/* ── 图片附加纯函数（rant 2026-09-02T15:23:53：GUI 粘贴/拖拽图片）── */ + +describe("imagePlaceholder / toSafeImageLabel", () => { + it("builds the TUI-identical literal placeholder", () => { + expect(imagePlaceholder("shot")).toBe("[📷 shot]"); + expect(imagePlaceholder("Image 1")).toBe("[📷 Image 1]"); + }); + + it("sanitizes labels like TUI safe_label (alnum/._- else _, cap 40, strip trailing ._)", () => { + expect(toSafeImageLabel("my photo")).toBe("my_photo"); + expect(toSafeImageLabel("a.b")).toBe("a.b"); + expect(toSafeImageLabel("x__.")).toBe("x"); // rstrip 语义同 TUI：清整段尾部 ._ + expect(toSafeImageLabel("!!!")).toBe("image"); + expect(toSafeImageLabel("")).toBe("image"); + expect(toSafeImageLabel("🙂")).toBe("image"); + expect(toSafeImageLabel("a".repeat(60)).length).toBe(40); + }); +}); + +describe("normalizePlaceholders", () => { + it("reverts tiptap-markdown backslash-escaped placeholders to literal text", () => { + expect(normalizePlaceholders("\\[📷 shot\\] hello", ["[📷 shot]"])).toBe("[📷 shot] hello"); + }); + + it("leaves already-literal placeholders untouched", () => { + expect(normalizePlaceholders("[📷 shot] hello", ["[📷 shot]"])).toBe("[📷 shot] hello"); + }); + + it("does not alter unrelated escaped brackets (only known labels)", () => { + expect(normalizePlaceholders("\\[keep\\] [📷 a]", ["[📷 a]"])).toBe("\\[keep\\] [📷 a]"); + }); + + it("handles multiple placeholders", () => { + const md = "\\[📷 a\\] mid \\[📷 b\\] end"; + expect(normalizePlaceholders(md, ["[📷 a]", "[📷 b]"])).toBe("[📷 a] mid [📷 b] end"); + }); +}); + +describe("resolveSendImages", () => { + const img = (label: string, path = `/p/${label}.png`, mime = "image/png") => ({ path, label, mime }); + + it("keeps images whose placeholder is still in the text, computing positions (TUI semantics)", () => { + const pending = [img("[📷 a]"), img("[📷 b]")]; + const out = resolveSendImages(pending, "lead [📷 b] tail [📷 a] end"); + // both kept; positions = indexOf literal label; sorted ascending + expect(out.map((i) => i.label)).toEqual(["[📷 b]", "[📷 a]"]); + expect(out[0].position).toBe("lead ".length); + expect(out[1].position).toBe("lead [📷 b] tail ".length); + }); + + it("drops images whose placeholder was deleted from the text", () => { + const pending = [img("[📷 kept]"), img("[📷 gone]")]; + const out = resolveSendImages(pending, "only [📷 kept] here"); + expect(out).toHaveLength(1); + expect(out[0].label).toBe("[📷 kept]"); + expect(out[0].position).toBe(5); + }); + + it("returns empty array when nothing remains", () => { + expect(resolveSendImages([img("[📷 a]")], "no placeholders here")).toEqual([]); + expect(resolveSendImages([], "text")).toEqual([]); + }); +}); diff --git a/emrg/gui/renderer/src/lib/composer.ts b/emrg/gui/renderer/src/lib/composer.ts index 985ebfd2..bdad3bb5 100644 --- a/emrg/gui/renderer/src/lib/composer.ts +++ b/emrg/gui/renderer/src/lib/composer.ts @@ -54,6 +54,8 @@ export interface QueuedSend { requestId: string; text: string; sandbox?: string | null; + /** 图片附件（rant 2026-09-02T15:23:53：busy 重发须携带原图，否则丢图） */ + images?: ImageAttach[] | null; } /** sendMessage busy 时入队（vanilla：queuedSends 不存在则建 [] 再 push） */ @@ -120,3 +122,65 @@ export function trackAfterResend( export function clearQueued(queuedSends: Map, sid: string): boolean { return queuedSends.delete(sid); } + +/* ── 图片附加（rant 2026-09-02T15:23:53：GUI 粘贴/拖拽图片，语义对齐 TUI）── */ + +/** + * 输入图元数据（对齐 TUI _pending_images 语义）： + * - label = **完整占位符文本**（"[📷 …]"）——TUI 发送前以 `label in text` 判活、 + * daemon 非 vision 降级文案直接引用该字段。 + * - position 在发送时计算（= 序列化 markdown 中占位符的字符偏移）。 + */ +export interface ImageAttach { + path: string; + label: string; + position?: number; + mime?: string; +} + +/** 构建占位符文本（双端一致的字面量：TUI 用 f"[📷 {label}]"） */ +export function imagePlaceholder(display: string): string { + return `[📷 ${display}]`; +} + +/** + * 文件名/展示标签清洗（镜像 TUI safe_label 规则）：非 [a-zA-Z0-9._-] → "_"， + * 截 40 字符、去尾 ._、空则回落 "image"。 + */ +export function toSafeImageLabel(raw: string): string { + const cleaned = raw + .split("") + .map((c) => (c.match(/[a-zA-Z0-9._-]/) ? c : "_")) + .join("") + .slice(0, 40) + .replace(/[._]+$/, ""); + return cleaned || "image"; +} + +/** + * tiptap-markdown 序列化会把字面占位符转义为 `\[📷 …\]`（防被当作链接/图片语法）， + * 发送前还原已知标签的字面量——保证 daemon 收到的文本与 TUI 完全一致（位置/判活 + * 都以还原后的字面量占位符为基准）。 + */ +export function normalizePlaceholders(md: string, labels: string[]): string { + let out = md; + for (const label of labels) { + const escaped = label.replace(/[\[\]]/g, (ch) => `\\${ch}`); + if (escaped !== label) out = out.split(escaped).join(label); + } + return out; +} + +/** + * 发送前收敛 pending 图片：占位符仍在文本中的保留（删了占位符即丢弃，TUI 同语义）， + * position = 字面占位符在文本中的字符偏移；返回按 position 升序的新数组。 + */ +export function resolveSendImages(pending: ImageAttach[], text: string): ImageAttach[] { + const out: ImageAttach[] = []; + for (const img of pending) { + const pos = text.indexOf(img.label); + if (pos >= 0) out.push({ ...img, position: pos }); + } + return out.sort((a, b) => (a.position ?? -1) - (b.position ?? -1)); +} + diff --git a/emrg/gui/renderer/src/lib/daemonBridge.ts b/emrg/gui/renderer/src/lib/daemonBridge.ts index 0f5e8f00..87114461 100644 --- a/emrg/gui/renderer/src/lib/daemonBridge.ts +++ b/emrg/gui/renderer/src/lib/daemonBridge.ts @@ -16,6 +16,7 @@ import type { TranslateFn } from "./utils"; import { createSnapshotStore, type SnapshotStore } from "./snapshot-store"; import type { TranscriptStore, DeltaChunk, DoneData, ToolStartData, ToolEndData } from "./transcript"; import type { OpenSessionEntry } from "./sidebar"; +import type { ImageAttach } from "./composer"; /* ── 事件帧（与 main.js 广播契约一致：{ type, data, sid }） ── */ @@ -128,6 +129,8 @@ export interface SendMessagePayload { text: string; requestId?: string; sandbox?: string; + /** 图片附件（rant 2026-09-02T15:23:53：透传 sendTask → daemon vision） */ + images?: ImageAttach[] | null; } /** window.emrg.init() 返回值（main.js emrg:init 处理器 → preload.init） */ @@ -173,7 +176,7 @@ export function createDaemonBridge(deps: DaemonBridgeDeps): DaemonBridge { // P2 queue-injection（#655）：busy 时发送的消息入 daemon 队列，queued_requeue // 以原 requestId 重发（不重加用户行）。逐 sid 记录（后台会话独立跟踪）。 - const queuedSends = new Map(); + const queuedSends = new Map(); function sidBusy(sid: string | null, busy: boolean): void { const k = KEY(sid); @@ -231,6 +234,7 @@ export function createDaemonBridge(deps: DaemonBridgeDeps): DaemonBridge { text: item.text, requestId: item.requestId, sandbox: item.sandbox, + ...(item.images ? { images: item.images } : {}), }); sidOwnRid(sid, res?.requestId ?? item.requestId); } catch { diff --git a/emrg/gui/test/preload-api.test.js b/emrg/gui/test/preload-api.test.js index 251cccab..a37ff8cc 100644 --- a/emrg/gui/test/preload-api.test.js +++ b/emrg/gui/test/preload-api.test.js @@ -3,12 +3,12 @@ * preload-api.test.js — window.emrg preload 桥 API 面守卫测试（React 迁移红线 #1）。 * * 背景（cycle cyc20260826-131007 发现）：React 迁移设计文档 §2.2 声称 preload - * API 46 个方法，实际 preload.js 已增长到 53 个 invoke 方法 + onEvent 订阅 + * API 46 个方法，实际 preload.js 已增长到 54 个 invoke 方法 + onEvent 订阅 * （设计文档口径过时）。此前 build-config.test.js 只抽查 listFiles/readFile 两个 * API——没有完整 API 面的回归守卫。React 化迁移（Batch 0-5）的红线就是 * "window.emrg 方法集合原样保留"，本测试把整个契约钉死： * - * 1. 全部 53 个 invoke 方法 + onEvent 必须存在（防误删/改名） + * 1. 全部 54 个 invoke 方法 + onEvent 必须存在（防误删/改名） * 2. 每个方法必须调用 ipcRenderer.invoke("emrg:<同名频道>")（防频道漂移） * 3. onEvent 必须订阅 emrg:event 广播（防事件通道漂移） * @@ -77,6 +77,7 @@ const EXPECTED_INVOKE_METHODS = [ "getSettings", "cancel", "pickProjectDir", + "saveImage", "log", ]; @@ -116,8 +117,8 @@ function parseApiBlock(block) { const api = parseApiBlock(extractApiBlock()); -test("window.emrg exposes the full 54-member contract (53 invoke + onEvent)", () => { - // 53 invoke 方法全部存在（防误删/改名） +test("window.emrg exposes the full 55-member contract (54 invoke + onEvent)", () => { + // 54 invoke 方法全部存在（防误删/改名） for (const name of EXPECTED_INVOKE_METHODS) { assert.ok(api.invoke.has(name), `缺少 invoke 方法 ${name}`); } diff --git a/emrg/server/daemon.py b/emrg/server/daemon.py index b6e0efc7..e572e307 100644 --- a/emrg/server/daemon.py +++ b/emrg/server/daemon.py @@ -2247,12 +2247,15 @@ def _build_user_content(text: str, images: list[dict] | None, vision: bool = Fal if chunk.strip(): content.append({"type": "text", "text": chunk}) last_pos = pos - # Encode image + # Encode image (rant 2026-09-02T15:23:53: GUI drag-in JPEG was + # previously hardcoded to image/png → wrong mime; mime metadata now + # drives the data URL, defaulting to png for TUI/history records) try: + mime = img.get("mime") or "image/png" b64 = base64.b64encode(Path(img["path"]).read_bytes()).decode() content.append({ "type": "image_url", - "image_url": {"url": f"data:image/png;base64,{b64}"}, + "image_url": {"url": f"data:{mime};base64,{b64}"}, }) except (OSError, FileNotFoundError) as e: logger.warning("failed to read image %s: %s", img.get("path"), e) diff --git a/tests/test_daemon.py b/tests/test_daemon.py index 8f87983b..d6b5a609 100644 --- a/tests/test_daemon.py +++ b/tests/test_daemon.py @@ -2795,6 +2795,51 @@ def test_tool_content_for_llm_missing_image(tmp_path): assert "Image unavailable" in out +# ── _build_user_content images (rant 2026-09-02T15:23:53: GUI image input) ── + + +def test_build_user_content_mime_metadata(tmp_path): + """images[].mime drives the data-URL mime (GUI drag-in JPEG previously + hardcoded to image/png → wrong mime; default stays png for TUI/history).""" + import base64 as _b64 + + server = _make_server() + server.llm.config.vision = True + jpg = tmp_path / "pic.jpg" + jpg.write_bytes(b"\xff\xd8\xff\xe0" + b"\x00" * 64) + + # mime metadata present → data URL uses it + out = EmrgServer._build_user_content( + "[📷 pic] hi", + [{"path": str(jpg), "label": "[📷 pic]", "position": 0, "mime": "image/jpeg"}], + vision=True, + ) + assert isinstance(out, list) + urls = [p["image_url"]["url"] for p in out if p.get("type") == "image_url"] + assert urls and urls[0].startswith("data:image/jpeg;base64,"), urls + assert _b64.b64decode(urls[0].split(",", 1)[1]) == jpg.read_bytes() + + # no mime → png default (back-compat with TUI/history records) + out2 = EmrgServer._build_user_content( + "plain", [{"path": str(jpg), "label": "[📷 pic]", "position": 0}], vision=True + ) + assert isinstance(out2, list) + urls2 = [p["image_url"]["url"] for p in out2 if p.get("type") == "image_url"] + assert urls2 and urls2[0].startswith("data:image/png;base64,"), urls2 + + +def test_build_user_content_non_vision_degrade(): + """vision=False → images degrade to a text placeholder, message text kept.""" + out = EmrgServer._build_user_content( + "hello", + [{"path": "/tmp/x.png", "label": "[📷 shot]", "position": 0}], + vision=False, + ) + assert isinstance(out, str) + assert "[📷 shot]" in out + assert "hello" in out + + # ── Daemon observability (rant 2026-08-25T09:25:32 — silent death) ──