Add bidirectional clipboard sync between Xvfb and browser

X11 → Browser:
- Background task polls xclip every 500ms for clipboard changes
- When content changes, sends {"type":"clipboard","data":"..."} via WS
- Browser receives and writes to navigator.clipboard

Browser → X11:
- Ctrl+V reads navigator.clipboard.readText() first
- Sends clipboard content to backend via WS message
- Backend writes to X11 clipboard via xclip -selection clipboard
- Then forwards the Ctrl+V keystroke so the app pastes

This allows copying links/text from GUI apps (like Nextcloud auth URLs)
and pasting content from the host browser into the GUI app.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
roberts 2026-03-15 01:26:42 -05:00
parent a5a7b01fd9
commit 36eff81111
3 changed files with 105 additions and 0 deletions

View file

@ -41,6 +41,7 @@ def _find_bin(name: str) -> Optional[str]:
XVFB_BIN = _find_bin("Xvfb") XVFB_BIN = _find_bin("Xvfb")
XDOTOOL_BIN = _find_bin("xdotool") XDOTOOL_BIN = _find_bin("xdotool")
IMPORT_BIN = _find_bin("import") # ImageMagick IMPORT_BIN = _find_bin("import") # ImageMagick
XCLIP_BIN = _find_bin("xclip")
HAS_DISPLAY_DEPS = all((XVFB_BIN, XDOTOOL_BIN, IMPORT_BIN)) HAS_DISPLAY_DEPS = all((XVFB_BIN, XDOTOOL_BIN, IMPORT_BIN))
@ -98,6 +99,8 @@ class ManagedGuiApp:
_websockets: list = field(default_factory=list, repr=False) _websockets: list = field(default_factory=list, repr=False)
_streaming: bool = field(default=False, repr=False) _streaming: bool = field(default=False, repr=False)
_focused: bool = field(default=False, repr=False) _focused: bool = field(default=False, repr=False)
_clipboard_task: Optional[asyncio.Task] = field(default=None, repr=False)
_last_clipboard: str = field(default="", repr=False)
@property @property
def alive(self) -> bool: def alive(self) -> bool:
@ -135,6 +138,8 @@ class ManagedGuiApp:
self._capture_task = asyncio.create_task(self._capture_loop()) self._capture_task = asyncio.create_task(self._capture_loop())
if self._stderr_task is None or self._stderr_task.done(): if self._stderr_task is None or self._stderr_task.done():
self._stderr_task = asyncio.create_task(self._read_stderr()) self._stderr_task = asyncio.create_task(self._read_stderr())
if XCLIP_BIN and (self._clipboard_task is None or self._clipboard_task.done()):
self._clipboard_task = asyncio.create_task(self._clipboard_loop())
async def _read_stderr(self): async def _read_stderr(self):
"""Read stderr in background, keep last N lines for exit diagnostics.""" """Read stderr in background, keep last N lines for exit diagnostics."""
@ -154,6 +159,62 @@ class ManagedGuiApp:
except Exception: except Exception:
pass pass
async def _clipboard_loop(self):
"""Poll X11 clipboard and push changes to connected browsers."""
if not XCLIP_BIN:
return
env = self._display_env()
while self.alive:
try:
await asyncio.sleep(0.5)
if not self._websockets:
continue
# Read clipboard from X11
proc = await asyncio.create_subprocess_exec(
XCLIP_BIN, "-selection", "clipboard", "-o",
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
env=env,
)
stdout, _ = await asyncio.wait_for(proc.communicate(), timeout=2)
if proc.returncode != 0 or not stdout:
continue
text = stdout.decode(errors="replace")
if text and text != self._last_clipboard:
self._last_clipboard = text
log.debug("Clipboard changed for %s: %d chars", self.app_id, len(text))
dead = []
for ws in self._websockets:
try:
await ws.send_json({"type": "clipboard", "data": text})
except Exception:
dead.append(ws)
for ws in dead:
self.detach_ws(ws)
except asyncio.TimeoutError:
continue
except Exception:
await asyncio.sleep(2)
async def set_clipboard(self, text: str):
"""Write text to the X11 clipboard from the browser."""
if not XCLIP_BIN:
return
env = self._display_env()
try:
proc = await asyncio.create_subprocess_exec(
XCLIP_BIN, "-selection", "clipboard",
stdin=asyncio.subprocess.PIPE,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
env=env,
)
await asyncio.wait_for(proc.communicate(input=text.encode()), timeout=2)
self._last_clipboard = text # prevent echo back
log.debug("Clipboard set from browser for %s: %d chars", self.app_id, len(text))
except Exception as e:
log.warning("Failed to set clipboard for %s: %s", self.app_id, e)
async def _capture_loop(self): async def _capture_loop(self):
"""Background: capture window pixmap → JPEG → fan-out.""" """Background: capture window pixmap → JPEG → fan-out."""
# Wait for window to appear (up to ~60s for heavy apps like Electron) # Wait for window to appear (up to ~60s for heavy apps like Electron)
@ -539,6 +600,8 @@ class ManagedGuiApp:
def kill(self): def kill(self):
if self._capture_task and not self._capture_task.done(): if self._capture_task and not self._capture_task.done():
self._capture_task.cancel() self._capture_task.cancel()
if self._clipboard_task and not self._clipboard_task.done():
self._clipboard_task.cancel()
if self.alive: if self.alive:
try: try:
self.process.terminate() self.process.terminate()

View file

@ -313,6 +313,11 @@ async def display_ws(
log.info("Input #%d from %s: %s %s", _input_count, username, log.info("Input #%d from %s: %s %s", _input_count, username,
msg_type, msg.get("action", msg.get("key", ""))) msg_type, msg.get("action", msg.get("key", "")))
await app.send_input(msg) await app.send_input(msg)
elif msg_type == "clipboard":
# Browser → X11 clipboard
text = msg.get("data", "")
if text:
await app.set_clipboard(text)
elif msg_type == "set_fps": elif msg_type == "set_fps":
fps = msg.get("fps", 10) fps = msg.get("fps", 10)
app.target_fps = max(1, min(30, fps)) app.target_fps = max(1, min(30, fps))

View file

@ -184,6 +184,16 @@
showStatus(state, 'exited', msg.data || 'Application exited'); showStatus(state, 'exited', msg.data || 'Application exited');
} else if (msg.type === 'error') { } else if (msg.type === 'error') {
showStatus(state, 'error', msg.data || 'Error'); showStatus(state, 'error', msg.data || 'Error');
} else if (msg.type === 'clipboard') {
// X11 clipboard → browser clipboard
const text = msg.data || '';
if (text && navigator.clipboard && navigator.clipboard.writeText) {
navigator.clipboard.writeText(text).then(() => {
console.log('[display] Clipboard synced to browser:', text.length, 'chars');
}).catch(err => {
console.warn('[display] Clipboard write failed:', err);
});
}
} else if (msg.type === 'meta') { } else if (msg.type === 'meta') {
// Could update title etc. // Could update title etc.
} }
@ -285,6 +295,33 @@
canvas.addEventListener('keydown', (e) => { canvas.addEventListener('keydown', (e) => {
e.preventDefault(); e.preventDefault();
e.stopPropagation(); e.stopPropagation();
// Ctrl+V / Cmd+V — sync browser clipboard to X11, then forward keystroke
if ((e.ctrlKey || e.metaKey) && e.key === 'v') {
if (navigator.clipboard && navigator.clipboard.readText) {
navigator.clipboard.readText().then(text => {
if (text) {
// Send clipboard content to X11 first
send({ type: 'clipboard', data: text });
}
// Then forward the Ctrl+V keystroke
send({
type: 'key', action: 'press',
key: e.key, code: e.code,
modifiers: getModifiers(e),
});
}).catch(() => {
// Clipboard read failed — just send the keystroke
send({
type: 'key', action: 'press',
key: e.key, code: e.code,
modifiers: getModifiers(e),
});
});
return;
}
}
send({ send({
type: 'key', action: 'press', type: 'key', action: 'press',
key: e.key, code: e.code, key: e.key, code: e.code,