Complete rewrite of input forwarding — fix mouse and keyboard events
Root causes fixed: - _focused was a stray class annotation in @dataclass, causing field ordering issues — moved to proper dataclass field - xdotool type --window WID not supported on all versions — removed --window flag, use focused window instead - xdotool commands with --window may fail silently — switched to absolute coordinates (window is at 0,0 filling the display) - All xdotool errors were silently swallowed — now logged with stderr Mouse events: - Use absolute mousemove + click (no --window) since window fills display - Separate mousemove and click into two calls for reliability - Fire-and-forget for mousemove to reduce latency Keyboard events: - xdotool type (no --window) for printable characters - xdotool key (no --window) for special keys and modifier combos - Window focused once via _ensure_focus, not per-event Diagnostics: - Backend logs first 5 input events received per WebSocket session - Backend logs xdotool stderr on failure - Frontend logs first 10 input events sent + WS state warnings - Frontend uses capture phase for keyboard events Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
parent
8d2a228599
commit
a5a7b01fd9
3 changed files with 107 additions and 83 deletions
|
|
@ -97,6 +97,7 @@ class ManagedGuiApp:
|
||||||
_stderr_lines: list = field(default_factory=list, repr=False)
|
_stderr_lines: list = field(default_factory=list, repr=False)
|
||||||
_websockets: list = field(default_factory=list, repr=False)
|
_websockets: list = field(default_factory=list, repr=False)
|
||||||
_streaming: bool = field(default=False, repr=False)
|
_streaming: bool = field(default=False, repr=False)
|
||||||
|
_focused: bool = field(default=False, repr=False)
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def alive(self) -> bool:
|
def alive(self) -> bool:
|
||||||
|
|
@ -365,7 +366,6 @@ class ManagedGuiApp:
|
||||||
return
|
return
|
||||||
env = self._display_env()
|
env = self._display_env()
|
||||||
wid = str(self.window_id)
|
wid = str(self.window_id)
|
||||||
try:
|
|
||||||
# Activate (focus) the window
|
# Activate (focus) the window
|
||||||
await self._xdotool("windowactivate", "--sync", wid, env=env)
|
await self._xdotool("windowactivate", "--sync", wid, env=env)
|
||||||
# Move to origin
|
# Move to origin
|
||||||
|
|
@ -373,13 +373,14 @@ class ManagedGuiApp:
|
||||||
# Resize to fill display (Xvfb is 1280x1024)
|
# Resize to fill display (Xvfb is 1280x1024)
|
||||||
await self._xdotool("windowsize", "--sync", wid, "1280", "1024", env=env)
|
await self._xdotool("windowsize", "--sync", wid, "1280", "1024", env=env)
|
||||||
# Focus the window for input
|
# Focus the window for input
|
||||||
await self._xdotool("windowfocus", wid, env=env)
|
rc = await self._xdotool("windowfocus", wid, env=env)
|
||||||
|
if rc == 0:
|
||||||
self._focused = True
|
self._focused = True
|
||||||
|
log.info("Window %s maximized and focused for %s", wid, self.app_id)
|
||||||
|
else:
|
||||||
|
log.warning("Window %s maximize completed but focus failed for %s", wid, self.app_id)
|
||||||
# Some apps need a moment to redraw after resize
|
# Some apps need a moment to redraw after resize
|
||||||
await asyncio.sleep(0.3)
|
await asyncio.sleep(0.3)
|
||||||
log.debug("Maximized window %s to 1280x1024 for %s", wid, self.app_id)
|
|
||||||
except Exception as e:
|
|
||||||
log.debug("Failed to maximize window %s: %s", wid, e)
|
|
||||||
|
|
||||||
async def _xdotool_search(self, *args, env=None) -> Optional[int]:
|
async def _xdotool_search(self, *args, env=None) -> Optional[int]:
|
||||||
"""Run xdotool search and return first window ID, or None."""
|
"""Run xdotool search and return first window ID, or None."""
|
||||||
|
|
@ -400,15 +401,17 @@ class ManagedGuiApp:
|
||||||
|
|
||||||
# ---- Input forwarding ----
|
# ---- Input forwarding ----
|
||||||
|
|
||||||
_focused: bool = False # tracks whether we've activated the window
|
|
||||||
|
|
||||||
async def _ensure_focus(self, env: dict):
|
async def _ensure_focus(self, env: dict):
|
||||||
"""Activate/focus the window once, then skip on subsequent calls."""
|
"""Activate/focus the window once, then skip on subsequent calls."""
|
||||||
if not self._focused:
|
if not self._focused:
|
||||||
wid = str(self.window_id)
|
wid = str(self.window_id)
|
||||||
await self._xdotool("windowactivate", wid, env=env)
|
rc1 = await self._xdotool("windowactivate", wid, env=env)
|
||||||
await self._xdotool("windowfocus", wid, env=env)
|
rc2 = await self._xdotool("windowfocus", wid, env=env)
|
||||||
|
if rc1 == 0 and rc2 == 0:
|
||||||
self._focused = True
|
self._focused = True
|
||||||
|
log.info("Window %s focused for input (app %s)", wid, self.app_id)
|
||||||
|
else:
|
||||||
|
log.warning("Failed to focus window %s (activate=%s, focus=%s)", wid, rc1, rc2)
|
||||||
|
|
||||||
async def send_input(self, msg: dict):
|
async def send_input(self, msg: dict):
|
||||||
"""Forward mouse/keyboard input to the X11 window."""
|
"""Forward mouse/keyboard input to the X11 window."""
|
||||||
|
|
@ -423,42 +426,30 @@ class ManagedGuiApp:
|
||||||
elif msg_type == "key":
|
elif msg_type == "key":
|
||||||
await self._handle_key(msg, env)
|
await self._handle_key(msg, env)
|
||||||
except Exception:
|
except Exception:
|
||||||
log.debug("Input forwarding error for %s", self.app_id, exc_info=True)
|
log.warning("Input forwarding error for %s", self.app_id, exc_info=True)
|
||||||
|
|
||||||
async def _handle_mouse(self, msg: dict, env: dict):
|
async def _handle_mouse(self, msg: dict, env: dict):
|
||||||
action = msg.get("action", "")
|
action = msg.get("action", "")
|
||||||
x, y = str(msg.get("x", 0)), str(msg.get("y", 0))
|
x, y = str(msg.get("x", 0)), str(msg.get("y", 0))
|
||||||
wid = str(self.window_id)
|
|
||||||
|
|
||||||
if action == "click":
|
if action == "click":
|
||||||
await self._ensure_focus(env)
|
await self._ensure_focus(env)
|
||||||
btn = str(msg.get("button", 1))
|
btn = str(msg.get("button", 1))
|
||||||
await self._xdotool(
|
# Use absolute coordinates — window is at 0,0 filling display
|
||||||
"mousemove", "--window", wid, x, y,
|
await self._xdotool("mousemove", x, y, env=env)
|
||||||
"click", "--window", wid, btn,
|
await self._xdotool("click", btn, env=env)
|
||||||
env=env,
|
|
||||||
)
|
|
||||||
elif action == "dblclick":
|
elif action == "dblclick":
|
||||||
await self._ensure_focus(env)
|
await self._ensure_focus(env)
|
||||||
btn = str(msg.get("button", 1))
|
btn = str(msg.get("button", 1))
|
||||||
await self._xdotool(
|
await self._xdotool("mousemove", x, y, env=env)
|
||||||
"mousemove", "--window", wid, x, y,
|
await self._xdotool("click", "--repeat", "2", btn, env=env)
|
||||||
"click", "--window", wid, "--repeat", "2", btn,
|
|
||||||
env=env,
|
|
||||||
)
|
|
||||||
elif action == "move":
|
elif action == "move":
|
||||||
await self._xdotool_fire(
|
self._xdotool_fire("mousemove", x, y, env=env)
|
||||||
"mousemove", "--window", wid, x, y,
|
|
||||||
env=env,
|
|
||||||
)
|
|
||||||
elif action == "scroll":
|
elif action == "scroll":
|
||||||
delta = msg.get("delta", 0)
|
delta = msg.get("delta", 0)
|
||||||
btn = "4" if delta < 0 else "5" # X11: 4=up, 5=down
|
btn = "4" if delta < 0 else "5" # X11: 4=up, 5=down
|
||||||
await self._xdotool(
|
await self._xdotool("mousemove", x, y, env=env)
|
||||||
"mousemove", "--window", wid, x, y,
|
await self._xdotool("click", btn, env=env)
|
||||||
"click", "--window", wid, btn,
|
|
||||||
env=env,
|
|
||||||
)
|
|
||||||
|
|
||||||
async def _handle_key(self, msg: dict, env: dict):
|
async def _handle_key(self, msg: dict, env: dict):
|
||||||
action = msg.get("action", "press")
|
action = msg.get("action", "press")
|
||||||
|
|
@ -474,7 +465,6 @@ class ManagedGuiApp:
|
||||||
return
|
return
|
||||||
|
|
||||||
await self._ensure_focus(env)
|
await self._ensure_focus(env)
|
||||||
wid = str(self.window_id)
|
|
||||||
|
|
||||||
# Determine which modifiers are held (excluding Shift for printable chars)
|
# Determine which modifiers are held (excluding Shift for printable chars)
|
||||||
active_mods = []
|
active_mods = []
|
||||||
|
|
@ -484,12 +474,9 @@ class ManagedGuiApp:
|
||||||
active_mods.append(xmod)
|
active_mods.append(xmod)
|
||||||
|
|
||||||
# Single printable character with no ctrl/alt/super modifiers → use xdotool type
|
# Single printable character with no ctrl/alt/super modifiers → use xdotool type
|
||||||
# This handles all special characters (@, #, !, etc.) correctly
|
# xdotool type handles @, #, !, etc. correctly via keyboard simulation
|
||||||
if len(key) == 1 and not active_mods:
|
if len(key) == 1 and not active_mods:
|
||||||
await self._xdotool(
|
await self._xdotool("type", "--delay", "0", "--clearmodifiers", key, env=env)
|
||||||
"type", "--delay", "0", "--clearmodifiers", "--window", wid, key,
|
|
||||||
env=env,
|
|
||||||
)
|
|
||||||
return
|
return
|
||||||
|
|
||||||
# Non-printable key or key combo with modifiers → use xdotool key
|
# Non-printable key or key combo with modifiers → use xdotool key
|
||||||
|
|
@ -507,37 +494,43 @@ class ManagedGuiApp:
|
||||||
if mod_parts:
|
if mod_parts:
|
||||||
xkey = "+".join(mod_parts) + "+" + xkey
|
xkey = "+".join(mod_parts) + "+" + xkey
|
||||||
|
|
||||||
await self._xdotool("key", "--window", wid, xkey, env=env)
|
await self._xdotool("key", xkey, env=env)
|
||||||
|
|
||||||
async def _xdotool(self, *args, env=None):
|
async def _xdotool(self, *args, env=None) -> int:
|
||||||
"""Run xdotool and wait for completion."""
|
"""Run xdotool and wait for completion. Returns exit code."""
|
||||||
|
try:
|
||||||
proc = await asyncio.create_subprocess_exec(
|
proc = await asyncio.create_subprocess_exec(
|
||||||
XDOTOOL_BIN, *args,
|
XDOTOOL_BIN, *args,
|
||||||
stdout=asyncio.subprocess.PIPE,
|
stdout=asyncio.subprocess.PIPE,
|
||||||
stderr=asyncio.subprocess.PIPE,
|
stderr=asyncio.subprocess.PIPE,
|
||||||
env=env,
|
env=env,
|
||||||
)
|
)
|
||||||
await asyncio.wait_for(proc.communicate(), timeout=5)
|
stdout, stderr = await asyncio.wait_for(proc.communicate(), timeout=5)
|
||||||
|
if proc.returncode != 0:
|
||||||
|
err = stderr.decode(errors="replace").strip() if stderr else ""
|
||||||
|
log.warning("xdotool %s failed (rc=%d): %s", args[0] if args else "?", proc.returncode, err)
|
||||||
|
return proc.returncode
|
||||||
|
except asyncio.TimeoutError:
|
||||||
|
log.warning("xdotool %s timed out", args[0] if args else "?")
|
||||||
|
return -1
|
||||||
|
except Exception as e:
|
||||||
|
log.warning("xdotool %s exception: %s", args[0] if args else "?", e)
|
||||||
|
return -1
|
||||||
|
|
||||||
async def _xdotool_fire(self, *args, env=None):
|
def _xdotool_fire(self, *args, env=None):
|
||||||
"""Run xdotool without waiting — fire and forget for low-latency ops."""
|
"""Schedule xdotool without waiting — fire and forget for low-latency ops."""
|
||||||
|
asyncio.create_task(self._xdotool_fire_async(*args, env=env))
|
||||||
|
|
||||||
|
async def _xdotool_fire_async(self, *args, env=None):
|
||||||
|
"""Fire-and-forget xdotool execution."""
|
||||||
|
try:
|
||||||
proc = await asyncio.create_subprocess_exec(
|
proc = await asyncio.create_subprocess_exec(
|
||||||
XDOTOOL_BIN, *args,
|
XDOTOOL_BIN, *args,
|
||||||
stdout=asyncio.subprocess.DEVNULL,
|
stdout=asyncio.subprocess.DEVNULL,
|
||||||
stderr=asyncio.subprocess.DEVNULL,
|
stderr=asyncio.subprocess.DEVNULL,
|
||||||
env=env,
|
env=env,
|
||||||
)
|
)
|
||||||
# Don't await — let it complete in background
|
|
||||||
asyncio.create_task(self._reap(proc))
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
async def _reap(proc):
|
|
||||||
"""Reap a fire-and-forget subprocess to avoid zombies."""
|
|
||||||
try:
|
|
||||||
await asyncio.wait_for(proc.wait(), timeout=5)
|
await asyncio.wait_for(proc.wait(), timeout=5)
|
||||||
except Exception:
|
|
||||||
try:
|
|
||||||
proc.kill()
|
|
||||||
except Exception:
|
except Exception:
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -301,12 +301,17 @@ async def display_ws(
|
||||||
app.attach_ws(websocket)
|
app.attach_ws(websocket)
|
||||||
log.info("WebSocket attached to app %s (%s) for %s", app_id, app.command, username)
|
log.info("WebSocket attached to app %s (%s) for %s", app_id, app.command, username)
|
||||||
|
|
||||||
|
_input_count = 0
|
||||||
try:
|
try:
|
||||||
while True:
|
while True:
|
||||||
msg = await websocket.receive_json()
|
msg = await websocket.receive_json()
|
||||||
msg_type = msg.get("type")
|
msg_type = msg.get("type")
|
||||||
|
|
||||||
if msg_type in ("mouse", "key"):
|
if msg_type in ("mouse", "key"):
|
||||||
|
_input_count += 1
|
||||||
|
if _input_count <= 5:
|
||||||
|
log.info("Input #%d from %s: %s %s", _input_count, username,
|
||||||
|
msg_type, msg.get("action", msg.get("key", "")))
|
||||||
await app.send_input(msg)
|
await app.send_input(msg)
|
||||||
elif msg_type == "set_fps":
|
elif msg_type == "set_fps":
|
||||||
fps = msg.get("fps", 10)
|
fps = msg.get("fps", 10)
|
||||||
|
|
|
||||||
|
|
@ -29,6 +29,8 @@
|
||||||
ws: null,
|
ws: null,
|
||||||
serverAppId: null, // backend app_id
|
serverAppId: null, // backend app_id
|
||||||
status: 'connecting',
|
status: 'connecting',
|
||||||
|
_lastMove: 0,
|
||||||
|
_inputCount: 0,
|
||||||
};
|
};
|
||||||
appState[guiConfig.id] = state;
|
appState[guiConfig.id] = state;
|
||||||
|
|
||||||
|
|
@ -52,6 +54,8 @@
|
||||||
canvas.width = 800;
|
canvas.width = 800;
|
||||||
canvas.height = 600;
|
canvas.height = 600;
|
||||||
canvas.tabIndex = 0;
|
canvas.tabIndex = 0;
|
||||||
|
// Ensure canvas is focusable and captures all events
|
||||||
|
canvas.style.outline = 'none';
|
||||||
wrap.appendChild(canvas);
|
wrap.appendChild(canvas);
|
||||||
container.appendChild(wrap);
|
container.appendChild(wrap);
|
||||||
|
|
||||||
|
|
@ -124,6 +128,7 @@
|
||||||
state.ws = ws;
|
state.ws = ws;
|
||||||
|
|
||||||
ws.onopen = () => {
|
ws.onopen = () => {
|
||||||
|
console.log('[display] WebSocket connected for', state.config.command);
|
||||||
showStatus(state, 'connecting', 'Waiting for window…');
|
showStatus(state, 'connecting', 'Waiting for window…');
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
@ -135,13 +140,14 @@
|
||||||
// JSON message
|
// JSON message
|
||||||
try {
|
try {
|
||||||
const msg = JSON.parse(e.data);
|
const msg = JSON.parse(e.data);
|
||||||
|
console.log('[display] WS message:', msg);
|
||||||
handleMessage(state, msg);
|
handleMessage(state, msg);
|
||||||
} catch (err) {}
|
} catch (err) {}
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
ws.onclose = () => {
|
ws.onclose = (e) => {
|
||||||
// Don't auto-reconnect — the app tab will re-init if reopened
|
console.log('[display] WebSocket closed:', e.code, e.reason);
|
||||||
};
|
};
|
||||||
|
|
||||||
ws.onerror = () => {
|
ws.onerror = () => {
|
||||||
|
|
@ -156,6 +162,7 @@
|
||||||
state.statusEl.style.display = 'none';
|
state.statusEl.style.display = 'none';
|
||||||
state.wrap.style.display = 'flex';
|
state.wrap.style.display = 'flex';
|
||||||
state.canvas.focus();
|
state.canvas.focus();
|
||||||
|
console.log('[display] First frame received — canvas visible, input active');
|
||||||
}
|
}
|
||||||
|
|
||||||
const blob = new Blob([buffer], { type: 'image/jpeg' });
|
const blob = new Blob([buffer], { type: 'image/jpeg' });
|
||||||
|
|
@ -164,6 +171,7 @@
|
||||||
if (state.canvas.width !== img.width || state.canvas.height !== img.height) {
|
if (state.canvas.width !== img.width || state.canvas.height !== img.height) {
|
||||||
state.canvas.width = img.width;
|
state.canvas.width = img.width;
|
||||||
state.canvas.height = img.height;
|
state.canvas.height = img.height;
|
||||||
|
console.log('[display] Canvas resized to', img.width, 'x', img.height);
|
||||||
}
|
}
|
||||||
state.ctx.drawImage(img, 0, 0);
|
state.ctx.drawImage(img, 0, 0);
|
||||||
URL.revokeObjectURL(img.src);
|
URL.revokeObjectURL(img.src);
|
||||||
|
|
@ -215,6 +223,15 @@
|
||||||
function send(msg) {
|
function send(msg) {
|
||||||
if (state.ws && state.ws.readyState === WebSocket.OPEN) {
|
if (state.ws && state.ws.readyState === WebSocket.OPEN) {
|
||||||
state.ws.send(JSON.stringify(msg));
|
state.ws.send(JSON.stringify(msg));
|
||||||
|
// Log first few events for debugging
|
||||||
|
state._inputCount++;
|
||||||
|
if (state._inputCount <= 10) {
|
||||||
|
console.log('[display] Input sent:', msg.type, msg.action || msg.key || '');
|
||||||
|
} else if (state._inputCount === 11) {
|
||||||
|
console.log('[display] (suppressing further input logs)');
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
console.warn('[display] Cannot send input — WS not open, state:', state.ws?.readyState);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -229,33 +246,42 @@
|
||||||
|
|
||||||
// Mouse events
|
// Mouse events
|
||||||
canvas.addEventListener('mousedown', (e) => {
|
canvas.addEventListener('mousedown', (e) => {
|
||||||
|
e.preventDefault();
|
||||||
|
e.stopPropagation();
|
||||||
canvas.focus();
|
canvas.focus();
|
||||||
const coords = scaleCoords(e);
|
const coords = scaleCoords(e);
|
||||||
send({ type: 'mouse', action: 'click', ...coords, button: e.button + 1 });
|
send({ type: 'mouse', action: 'click', ...coords, button: e.button + 1 });
|
||||||
});
|
});
|
||||||
|
|
||||||
canvas.addEventListener('dblclick', (e) => {
|
canvas.addEventListener('dblclick', (e) => {
|
||||||
|
e.preventDefault();
|
||||||
|
e.stopPropagation();
|
||||||
const coords = scaleCoords(e);
|
const coords = scaleCoords(e);
|
||||||
send({ type: 'mouse', action: 'dblclick', ...coords, button: e.button + 1 });
|
send({ type: 'mouse', action: 'dblclick', ...coords, button: e.button + 1 });
|
||||||
});
|
});
|
||||||
|
|
||||||
canvas.addEventListener('mousemove', (e) => {
|
canvas.addEventListener('mousemove', (e) => {
|
||||||
// Throttle mousemove to ~30fps
|
// Throttle mousemove to ~30fps
|
||||||
if (state._lastMove && Date.now() - state._lastMove < 33) return;
|
const now = Date.now();
|
||||||
state._lastMove = Date.now();
|
if (now - state._lastMove < 33) return;
|
||||||
|
state._lastMove = now;
|
||||||
const coords = scaleCoords(e);
|
const coords = scaleCoords(e);
|
||||||
send({ type: 'mouse', action: 'move', ...coords });
|
send({ type: 'mouse', action: 'move', ...coords });
|
||||||
});
|
});
|
||||||
|
|
||||||
canvas.addEventListener('wheel', (e) => {
|
canvas.addEventListener('wheel', (e) => {
|
||||||
e.preventDefault();
|
e.preventDefault();
|
||||||
|
e.stopPropagation();
|
||||||
const coords = scaleCoords(e);
|
const coords = scaleCoords(e);
|
||||||
send({ type: 'mouse', action: 'scroll', ...coords, delta: e.deltaY > 0 ? 1 : -1 });
|
send({ type: 'mouse', action: 'scroll', ...coords, delta: e.deltaY > 0 ? 1 : -1 });
|
||||||
}, { passive: false });
|
}, { passive: false });
|
||||||
|
|
||||||
canvas.addEventListener('contextmenu', (e) => e.preventDefault());
|
canvas.addEventListener('contextmenu', (e) => {
|
||||||
|
e.preventDefault();
|
||||||
|
e.stopPropagation();
|
||||||
|
});
|
||||||
|
|
||||||
// Keyboard events
|
// Keyboard events — capture phase to intercept before Atlus shell
|
||||||
canvas.addEventListener('keydown', (e) => {
|
canvas.addEventListener('keydown', (e) => {
|
||||||
e.preventDefault();
|
e.preventDefault();
|
||||||
e.stopPropagation();
|
e.stopPropagation();
|
||||||
|
|
@ -264,7 +290,7 @@
|
||||||
key: e.key, code: e.code,
|
key: e.key, code: e.code,
|
||||||
modifiers: getModifiers(e),
|
modifiers: getModifiers(e),
|
||||||
});
|
});
|
||||||
});
|
}, true); // capture phase
|
||||||
|
|
||||||
canvas.addEventListener('keyup', (e) => {
|
canvas.addEventListener('keyup', (e) => {
|
||||||
e.preventDefault();
|
e.preventDefault();
|
||||||
|
|
@ -274,7 +300,7 @@
|
||||||
key: e.key, code: e.code,
|
key: e.key, code: e.code,
|
||||||
modifiers: getModifiers(e),
|
modifiers: getModifiers(e),
|
||||||
});
|
});
|
||||||
});
|
}, true); // capture phase
|
||||||
}
|
}
|
||||||
|
|
||||||
// ---- App registration (no left dock — apps live in the right panel) ----
|
// ---- App registration (no left dock — apps live in the right panel) ----
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue