From 36eff81111d8c32f9141f756c09053dc84709b3f Mon Sep 17 00:00:00 2001 From: roberts Date: Sun, 15 Mar 2026 01:26:42 -0500 Subject: [PATCH] Add bidirectional clipboard sync between Xvfb and browser MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X11 → Browser: - Background task polls xclip every 500ms for clipboard changes - When content changes, sends {"type":"clipboard","data":"..."} via WS - Browser receives and writes to navigator.clipboard Browser → X11: - Ctrl+V reads navigator.clipboard.readText() first - Sends clipboard content to backend via WS message - Backend writes to X11 clipboard via xclip -selection clipboard - Then forwards the Ctrl+V keystroke so the app pastes This allows copying links/text from GUI apps (like Nextcloud auth URLs) and pasting content from the host browser into the GUI app. Co-Authored-By: Claude Opus 4.6 --- backend/display.py | 63 +++++++++++++++++++++++++++++++++++++ backend/routers/display.py | 5 +++ frontend/js/apps/display.js | 37 ++++++++++++++++++++++ 3 files changed, 105 insertions(+) diff --git a/backend/display.py b/backend/display.py index b00c89e..e6a73c5 100644 --- a/backend/display.py +++ b/backend/display.py @@ -41,6 +41,7 @@ def _find_bin(name: str) -> Optional[str]: XVFB_BIN = _find_bin("Xvfb") XDOTOOL_BIN = _find_bin("xdotool") IMPORT_BIN = _find_bin("import") # ImageMagick +XCLIP_BIN = _find_bin("xclip") HAS_DISPLAY_DEPS = all((XVFB_BIN, XDOTOOL_BIN, IMPORT_BIN)) @@ -98,6 +99,8 @@ class ManagedGuiApp: _websockets: list = field(default_factory=list, repr=False) _streaming: bool = field(default=False, repr=False) _focused: bool = field(default=False, repr=False) + _clipboard_task: Optional[asyncio.Task] = field(default=None, repr=False) + _last_clipboard: str = field(default="", repr=False) @property def alive(self) -> bool: @@ -135,6 +138,8 @@ class ManagedGuiApp: self._capture_task = asyncio.create_task(self._capture_loop()) if self._stderr_task is None or self._stderr_task.done(): self._stderr_task = asyncio.create_task(self._read_stderr()) + if XCLIP_BIN and (self._clipboard_task is None or self._clipboard_task.done()): + self._clipboard_task = asyncio.create_task(self._clipboard_loop()) async def _read_stderr(self): """Read stderr in background, keep last N lines for exit diagnostics.""" @@ -154,6 +159,62 @@ class ManagedGuiApp: except Exception: pass + async def _clipboard_loop(self): + """Poll X11 clipboard and push changes to connected browsers.""" + if not XCLIP_BIN: + return + env = self._display_env() + while self.alive: + try: + await asyncio.sleep(0.5) + if not self._websockets: + continue + # Read clipboard from X11 + proc = await asyncio.create_subprocess_exec( + XCLIP_BIN, "-selection", "clipboard", "-o", + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + env=env, + ) + stdout, _ = await asyncio.wait_for(proc.communicate(), timeout=2) + if proc.returncode != 0 or not stdout: + continue + text = stdout.decode(errors="replace") + if text and text != self._last_clipboard: + self._last_clipboard = text + log.debug("Clipboard changed for %s: %d chars", self.app_id, len(text)) + dead = [] + for ws in self._websockets: + try: + await ws.send_json({"type": "clipboard", "data": text}) + except Exception: + dead.append(ws) + for ws in dead: + self.detach_ws(ws) + except asyncio.TimeoutError: + continue + except Exception: + await asyncio.sleep(2) + + async def set_clipboard(self, text: str): + """Write text to the X11 clipboard from the browser.""" + if not XCLIP_BIN: + return + env = self._display_env() + try: + proc = await asyncio.create_subprocess_exec( + XCLIP_BIN, "-selection", "clipboard", + stdin=asyncio.subprocess.PIPE, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + env=env, + ) + await asyncio.wait_for(proc.communicate(input=text.encode()), timeout=2) + self._last_clipboard = text # prevent echo back + log.debug("Clipboard set from browser for %s: %d chars", self.app_id, len(text)) + except Exception as e: + log.warning("Failed to set clipboard for %s: %s", self.app_id, e) + async def _capture_loop(self): """Background: capture window pixmap → JPEG → fan-out.""" # Wait for window to appear (up to ~60s for heavy apps like Electron) @@ -539,6 +600,8 @@ class ManagedGuiApp: def kill(self): if self._capture_task and not self._capture_task.done(): self._capture_task.cancel() + if self._clipboard_task and not self._clipboard_task.done(): + self._clipboard_task.cancel() if self.alive: try: self.process.terminate() diff --git a/backend/routers/display.py b/backend/routers/display.py index 0534917..e9d4446 100644 --- a/backend/routers/display.py +++ b/backend/routers/display.py @@ -313,6 +313,11 @@ async def display_ws( log.info("Input #%d from %s: %s %s", _input_count, username, msg_type, msg.get("action", msg.get("key", ""))) await app.send_input(msg) + elif msg_type == "clipboard": + # Browser → X11 clipboard + text = msg.get("data", "") + if text: + await app.set_clipboard(text) elif msg_type == "set_fps": fps = msg.get("fps", 10) app.target_fps = max(1, min(30, fps)) diff --git a/frontend/js/apps/display.js b/frontend/js/apps/display.js index e06d470..7fa1be5 100644 --- a/frontend/js/apps/display.js +++ b/frontend/js/apps/display.js @@ -184,6 +184,16 @@ showStatus(state, 'exited', msg.data || 'Application exited'); } else if (msg.type === 'error') { showStatus(state, 'error', msg.data || 'Error'); + } else if (msg.type === 'clipboard') { + // X11 clipboard → browser clipboard + const text = msg.data || ''; + if (text && navigator.clipboard && navigator.clipboard.writeText) { + navigator.clipboard.writeText(text).then(() => { + console.log('[display] Clipboard synced to browser:', text.length, 'chars'); + }).catch(err => { + console.warn('[display] Clipboard write failed:', err); + }); + } } else if (msg.type === 'meta') { // Could update title etc. } @@ -285,6 +295,33 @@ canvas.addEventListener('keydown', (e) => { e.preventDefault(); e.stopPropagation(); + + // Ctrl+V / Cmd+V — sync browser clipboard to X11, then forward keystroke + if ((e.ctrlKey || e.metaKey) && e.key === 'v') { + if (navigator.clipboard && navigator.clipboard.readText) { + navigator.clipboard.readText().then(text => { + if (text) { + // Send clipboard content to X11 first + send({ type: 'clipboard', data: text }); + } + // Then forward the Ctrl+V keystroke + send({ + type: 'key', action: 'press', + key: e.key, code: e.code, + modifiers: getModifiers(e), + }); + }).catch(() => { + // Clipboard read failed — just send the keystroke + send({ + type: 'key', action: 'press', + key: e.key, code: e.code, + modifiers: getModifiers(e), + }); + }); + return; + } + } + send({ type: 'key', action: 'press', key: e.key, code: e.code,