diff --git a/backend/config.py b/backend/config.py index 899ce75..2517978 100644 --- a/backend/config.py +++ b/backend/config.py @@ -63,6 +63,7 @@ _DEFAULT_CONFIG: dict = { "terminal", "files", "services", "tasks", "network", "settings" ], "panel_services": [], # systemd unit names to show in panel + "gui_apps": [], # GUI apps: [{id, name, command, icon, args, target_fps}] "asi_bridge": { "cifs_share": "//192.168.10.120/share", "mount_point": "/mnt/asiair", diff --git a/backend/display.py b/backend/display.py new file mode 100644 index 0000000..8c59a96 --- /dev/null +++ b/backend/display.py @@ -0,0 +1,499 @@ +"""Atlus — GUI application display manager. + +Manages a virtual X11 display (Xvfb) and per-window frame capture +for streaming native GUI applications to the browser via WebSocket. +Each GUI app window is captured independently and streamed as JPEG frames. +""" + +import asyncio +import logging +import os +import re +import shutil +import signal +import time +import uuid +from dataclasses import dataclass, field +from typing import Optional + +log = logging.getLogger("atlus.display") + +# --------------------------------------------------------------------------- +# Binary discovery (systemd strips PATH) +# --------------------------------------------------------------------------- + +_SEARCH_PATHS = ("/usr/bin", "/usr/local/bin", "/bin", "/usr/sbin", "/sbin") + + +def _find_bin(name: str) -> Optional[str]: + found = shutil.which(name) + if found: + return found + for d in _SEARCH_PATHS: + p = os.path.join(d, name) + if os.path.isfile(p) and os.access(p, os.X_OK): + return p + return None + + +XVFB_BIN = _find_bin("Xvfb") +XDOTOOL_BIN = _find_bin("xdotool") +IMPORT_BIN = _find_bin("import") # ImageMagick + +HAS_DISPLAY_DEPS = all((XVFB_BIN, XDOTOOL_BIN, IMPORT_BIN)) + +# Allowed command pattern — alphanumeric + hyphens only +_SAFE_CMD = re.compile(r"^[a-zA-Z0-9][a-zA-Z0-9._\-]*$") + +# Key translation: DOM key names → xdotool key names +_KEY_MAP = { + "Control": "ctrl", + "Shift": "shift", + "Alt": "alt", + "Meta": "super", + "Enter": "Return", + "Backspace": "BackSpace", + "Delete": "Delete", + "Escape": "Escape", + "Tab": "Tab", + "ArrowUp": "Up", + "ArrowDown": "Down", + "ArrowLeft": "Left", + "ArrowRight": "Right", + "Home": "Home", + "End": "End", + "PageUp": "Prior", + "PageDown": "Next", + "Insert": "Insert", + " ": "space", + "F1": "F1", "F2": "F2", "F3": "F3", "F4": "F4", + "F5": "F5", "F6": "F6", "F7": "F7", "F8": "F8", + "F9": "F9", "F10": "F10", "F11": "F11", "F12": "F12", +} + + +# --------------------------------------------------------------------------- +# ManagedGuiApp — a GUI process with per-window frame capture +# --------------------------------------------------------------------------- + +@dataclass +class ManagedGuiApp: + """A GUI application running on a virtual X display.""" + + app_id: str + command: str + title: str + display_num: int + process: Optional[asyncio.subprocess.Process] = None + window_id: Optional[int] = None + target_fps: int = 10 + created_at: float = field(default_factory=time.time) + last_frame: Optional[bytes] = field(default=None, repr=False) + _capture_task: Optional[asyncio.Task] = field(default=None, repr=False) + _websockets: list = field(default_factory=list, repr=False) + _streaming: bool = field(default=False, repr=False) + + @property + def alive(self) -> bool: + return self.process is not None and self.process.returncode is None + + def _display_env(self) -> dict: + env = {**os.environ, "DISPLAY": f":{self.display_num}"} + # Ensure PATH has common binary dirs + path = env.get("PATH", "") + for p in _SEARCH_PATHS: + if p not in path: + path = p + ":" + path + env["PATH"] = path + return env + + # ---- WebSocket fan-out ---- + + def attach_ws(self, ws): + if ws not in self._websockets: + self._websockets.append(ws) + self._streaming = True + + def detach_ws(self, ws): + try: + self._websockets.remove(ws) + except ValueError: + pass + if not self._websockets: + self._streaming = False + + # ---- Frame capture ---- + + def start_capture(self): + if self._capture_task is None or self._capture_task.done(): + self._capture_task = asyncio.create_task(self._capture_loop()) + + async def _capture_loop(self): + """Background: capture window pixmap → JPEG → fan-out.""" + # Wait for window to appear + for attempt in range(20): + if not self.alive: + return + await self._discover_window() + if self.window_id: + break + await asyncio.sleep(0.5 * (1 + attempt * 0.2)) + + if not self.window_id: + log.warning("No window found for app %s (%s)", self.app_id, self.command) + # Notify any attached viewers + for ws in list(self._websockets): + try: + await ws.send_json({"type": "error", "data": "No window found for application"}) + except Exception: + pass + return + + log.info("Capture loop started for app %s window %d", self.app_id, self.window_id) + + while self.alive: + if not self._streaming or not self._websockets: + await asyncio.sleep(0.5) + continue + try: + frame = await self._capture_frame() + if frame: + self.last_frame = frame + dead = [] + for ws in self._websockets: + try: + await ws.send_bytes(frame) + except Exception: + dead.append(ws) + for ws in dead: + self.detach_ws(ws) + except Exception: + log.exception("Capture error for %s", self.app_id) + break + await asyncio.sleep(1.0 / self.target_fps) + + # Process exited — notify viewers + for ws in list(self._websockets): + try: + await ws.send_json({"type": "closed", "data": "Application exited"}) + except Exception: + pass + log.info("Capture loop ended for app %s", self.app_id) + + async def _capture_frame(self) -> Optional[bytes]: + """Capture window as JPEG using ImageMagick import.""" + if not self.window_id or not IMPORT_BIN: + return None + try: + proc = await asyncio.create_subprocess_exec( + IMPORT_BIN, "-window", str(self.window_id), + "-quality", "60", "jpeg:-", + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + env=self._display_env(), + ) + stdout, stderr = await asyncio.wait_for(proc.communicate(), timeout=5) + if proc.returncode == 0 and stdout: + return stdout + return None + except asyncio.TimeoutError: + return None + except Exception: + return None + + async def _discover_window(self): + """Find X11 window ID for this app's process.""" + if not XDOTOOL_BIN or not self.process: + return + try: + proc = await asyncio.create_subprocess_exec( + XDOTOOL_BIN, "search", "--pid", str(self.process.pid), + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + env=self._display_env(), + ) + stdout, _ = await asyncio.wait_for(proc.communicate(), timeout=5) + lines = stdout.decode().strip().splitlines() + if lines: + self.window_id = int(lines[0]) + log.debug("Discovered window %d for app %s (pid %d)", + self.window_id, self.app_id, self.process.pid) + except Exception: + pass + + # ---- Input forwarding ---- + + async def send_input(self, msg: dict): + """Forward mouse/keyboard input to the X11 window.""" + if not self.window_id or not XDOTOOL_BIN: + return + env = self._display_env() + msg_type = msg.get("type") + + try: + if msg_type == "mouse": + await self._handle_mouse(msg, env) + elif msg_type == "key": + await self._handle_key(msg, env) + except Exception: + log.debug("Input forwarding error for %s", self.app_id, exc_info=True) + + async def _handle_mouse(self, msg: dict, env: dict): + action = msg.get("action", "") + x, y = str(msg.get("x", 0)), str(msg.get("y", 0)) + wid = str(self.window_id) + + if action == "click": + btn = str(msg.get("button", 1)) + await self._xdotool( + "mousemove", "--window", wid, x, y, + "click", "--window", wid, btn, + env=env, + ) + elif action == "dblclick": + btn = str(msg.get("button", 1)) + await self._xdotool( + "mousemove", "--window", wid, x, y, + "click", "--window", wid, "--repeat", "2", btn, + env=env, + ) + elif action == "move": + await self._xdotool( + "mousemove", "--window", wid, x, y, + env=env, + ) + elif action == "scroll": + delta = msg.get("delta", 0) + btn = "4" if delta < 0 else "5" # X11: 4=up, 5=down + await self._xdotool( + "mousemove", "--window", wid, x, y, + "click", "--window", wid, btn, + env=env, + ) + + async def _handle_key(self, msg: dict, env: dict): + action = msg.get("action", "press") + key = msg.get("key", "") + modifiers = msg.get("modifiers", []) + + # Translate key name + xkey = _KEY_MAP.get(key, key) + + # Skip standalone modifier key events + if xkey in ("ctrl", "shift", "alt", "super"): + return + + # Build modifier prefix + mod_parts = [] + for m in modifiers: + xmod = _KEY_MAP.get(m, m.lower()) + if xmod not in ("ctrl", "shift", "alt", "super"): + continue + mod_parts.append(xmod) + + if mod_parts: + xkey = "+".join(mod_parts) + "+" + xkey + + wid = str(self.window_id) + if action == "press": + await self._xdotool("key", "--window", wid, xkey, env=env) + # release events handled implicitly by xdotool key + + async def _xdotool(self, *args, env=None): + proc = await asyncio.create_subprocess_exec( + XDOTOOL_BIN, *args, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + env=env, + ) + await asyncio.wait_for(proc.communicate(), timeout=5) + + # ---- Lifecycle ---- + + def kill(self): + if self._capture_task and not self._capture_task.done(): + self._capture_task.cancel() + if self.alive: + try: + self.process.terminate() + except Exception: + pass + + def to_dict(self) -> dict: + return { + "app_id": self.app_id, + "command": self.command, + "title": self.title, + "alive": self.alive, + "pid": self.process.pid if self.process else None, + "window_id": self.window_id, + "streaming": self._streaming, + "viewers": len(self._websockets), + "created_at": self.created_at, + } + + +# --------------------------------------------------------------------------- +# Display Manager — manages Xvfb displays and GUI apps +# --------------------------------------------------------------------------- + +class DisplayManager: + """Manages virtual X displays and GUI application lifecycle.""" + + def __init__(self): + self._xvfb_procs: dict[str, asyncio.subprocess.Process] = {} + self._display_nums: dict[str, int] = {} + self._apps: dict[str, dict[str, ManagedGuiApp]] = {} + self._next_display = 99 + + def _require_deps(self): + if not HAS_DISPLAY_DEPS: + missing = [] + if not XVFB_BIN: + missing.append("Xvfb") + if not XDOTOOL_BIN: + missing.append("xdotool") + if not IMPORT_BIN: + missing.append("import (ImageMagick)") + raise RuntimeError(f"Missing display dependencies: {', '.join(missing)}") + + async def get_or_create_display(self, username: str) -> int: + """Start Xvfb if needed, return display number.""" + self._require_deps() + + if username in self._display_nums: + # Check if Xvfb is still running + xvfb = self._xvfb_procs.get(username) + if xvfb and xvfb.returncode is None: + return self._display_nums[username] + # Xvfb died — restart + log.warning("Xvfb died for %s, restarting", username) + + display_num = self._next_display + self._next_display += 1 + + log.info("Starting Xvfb :%d for %s", display_num, username) + xvfb = await asyncio.create_subprocess_exec( + XVFB_BIN, f":{display_num}", + "-screen", "0", "1280x1024x24", + "-ac", # disable access control + "-nolisten", "tcp", + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + + # Wait briefly for Xvfb to start + await asyncio.sleep(0.5) + if xvfb.returncode is not None: + raise RuntimeError(f"Xvfb failed to start (rc={xvfb.returncode})") + + self._xvfb_procs[username] = xvfb + self._display_nums[username] = display_num + log.info("Xvfb :%d started for %s (pid %d)", display_num, username, xvfb.pid) + return display_num + + async def launch_app( + self, username: str, command: str, title: str = "", + args: list[str] | None = None, target_fps: int = 10, + ) -> ManagedGuiApp: + """Launch a GUI application on the user's virtual display.""" + # Validate command + if not _SAFE_CMD.match(command): + raise ValueError(f"Invalid command name: {command}") + + # Find the binary + cmd_bin = _find_bin(command) + if not cmd_bin: + raise FileNotFoundError(f"Command not found: {command}") + + display_num = await self.get_or_create_display(username) + + env = { + **os.environ, + "DISPLAY": f":{display_num}", + "HOME": os.path.expanduser("~"), + } + # Ensure PATH + path = env.get("PATH", "") + for p in _SEARCH_PATHS: + if p not in path: + path = p + ":" + path + env["PATH"] = path + + cmd = [cmd_bin] + (args or []) + log.info("Launching GUI app: %s (display :%d, user %s)", " ".join(cmd), display_num, username) + + process = await asyncio.create_subprocess_exec( + *cmd, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + env=env, + ) + + app_id = str(uuid.uuid4())[:8] + app = ManagedGuiApp( + app_id=app_id, + command=command, + title=title or command, + display_num=display_num, + process=process, + target_fps=target_fps, + ) + app.start_capture() + + self._apps.setdefault(username, {})[app_id] = app + log.info("GUI app %s launched: %s (pid %d)", app_id, command, process.pid) + return app + + def get_app(self, username: str, app_id: str) -> Optional[ManagedGuiApp]: + return self._apps.get(username, {}).get(app_id) + + def get_app_by_command(self, username: str, command: str) -> Optional[ManagedGuiApp]: + """Find a running app by command name.""" + for app in self._apps.get(username, {}).values(): + if app.command == command and app.alive: + return app + return None + + def close_app(self, username: str, app_id: str) -> bool: + user_apps = self._apps.get(username, {}) + app = user_apps.pop(app_id, None) + if not app: + return False + app.kill() + log.info("Closed GUI app %s (%s) for %s", app_id, app.command, username) + return True + + def list_apps(self, username: str) -> list[dict]: + user_apps = self._apps.get(username, {}) + # Prune dead apps + dead = [aid for aid, a in user_apps.items() if not a.alive] + for aid in dead: + app = user_apps.pop(aid) + app.kill() + return [a.to_dict() for a in user_apps.values()] + + async def shutdown_all(self): + """Kill all apps and Xvfb displays.""" + for username, apps in self._apps.items(): + for app in apps.values(): + app.kill() + self._apps.clear() + + for username, xvfb in self._xvfb_procs.items(): + if xvfb.returncode is None: + try: + xvfb.terminate() + await asyncio.wait_for(xvfb.wait(), timeout=5) + except Exception: + xvfb.kill() + self._xvfb_procs.clear() + self._display_nums.clear() + log.info("All displays shut down") + + +# --------------------------------------------------------------------------- +# Singleton +# --------------------------------------------------------------------------- + +display_manager = DisplayManager() diff --git a/backend/main.py b/backend/main.py index 98a7ca4..6f40a8d 100644 --- a/backend/main.py +++ b/backend/main.py @@ -11,8 +11,9 @@ from pydantic import BaseModel from backend.auth import authenticate_user, create_token, logout from backend.config import FRONTEND_DIR, HOST, PORT -from backend.routers import stats, terminal, files, services, processes, settings, network, packages, updates, session +from backend.routers import stats, terminal, files, services, processes, settings, network, packages, updates, session, display from backend.sessions import manager as session_manager +from backend.display import display_manager from backend.routers.plugins import asi_bridge logging.basicConfig( @@ -55,6 +56,8 @@ async def lifespan(app: FastAPI): # Kill all PTYs on shutdown session_manager.shutdown_all() + # Stop all virtual displays and GUI apps + await display_manager.shutdown_all() log.info("Atlus shutdown complete") @@ -109,6 +112,7 @@ app.include_router(network.router) app.include_router(packages.router) app.include_router(updates.router) app.include_router(session.router) +app.include_router(display.router) app.include_router(asi_bridge.router) diff --git a/backend/routers/display.py b/backend/routers/display.py new file mode 100644 index 0000000..eb2e869 --- /dev/null +++ b/backend/routers/display.py @@ -0,0 +1,161 @@ +"""Display — WebSocket frame streaming + REST app management for GUI apps. + +Pattern mirrors terminal.py: WebSocket attaches to a running GUI app, +receives JPEG frames, sends input events. REST endpoints manage app lifecycle. +""" + +import logging +from typing import Optional + +from fastapi import APIRouter, Depends, HTTPException, Query, WebSocket, WebSocketDisconnect +from pydantic import BaseModel + +from backend.auth import get_current_user, ws_authenticate +from backend.display import display_manager, HAS_DISPLAY_DEPS + +router = APIRouter(prefix="/api/display", tags=["display"]) +log = logging.getLogger("atlus.display.router") + + +# --------------------------------------------------------------------------- +# Guards +# --------------------------------------------------------------------------- + +def _require_deps(): + if not HAS_DISPLAY_DEPS: + raise HTTPException(503, "Display dependencies not installed (Xvfb, xdotool, ImageMagick)") + + +# --------------------------------------------------------------------------- +# REST — app lifecycle +# --------------------------------------------------------------------------- + +class AppLaunchRequest(BaseModel): + command: str + title: str = "" + args: list[str] = [] + target_fps: int = 10 + + +@router.get("/apps") +async def list_apps(user: str = Depends(get_current_user)): + """List running GUI apps for the current user.""" + _require_deps() + return {"apps": display_manager.list_apps(user)} + + +@router.post("/apps") +async def launch_app(req: AppLaunchRequest, user: str = Depends(get_current_user)): + """Launch a GUI application on the user's virtual display.""" + _require_deps() + + # Check if this command is already running + existing = display_manager.get_app_by_command(user, req.command) + if existing: + return existing.to_dict() + + try: + app = await display_manager.launch_app( + user, req.command, req.title, req.args, req.target_fps, + ) + return app.to_dict() + except ValueError as e: + raise HTTPException(400, str(e)) + except FileNotFoundError as e: + raise HTTPException(404, str(e)) + except RuntimeError as e: + raise HTTPException(500, str(e)) + + +@router.delete("/apps/{app_id}") +async def close_app(app_id: str, user: str = Depends(get_current_user)): + """Stop a running GUI application.""" + _require_deps() + if not display_manager.close_app(user, app_id): + raise HTTPException(404, "App not found") + return {"ok": True} + + +@router.get("/status") +async def display_status(user: str = Depends(get_current_user)): + """Check display system availability.""" + return { + "available": HAS_DISPLAY_DEPS, + "apps": display_manager.list_apps(user), + } + + +# --------------------------------------------------------------------------- +# WebSocket — frame streaming + input +# --------------------------------------------------------------------------- + +@router.websocket("/ws") +async def display_ws( + websocket: WebSocket, + app_id: str = Query(default=None), +): + """Attach to a GUI app's window — receive JPEG frames, send input. + + Query params: + - app_id: required — the GUI app to stream + - token: auth token (handled by ws_authenticate) + + Server sends: + - Binary: raw JPEG frame bytes + - {"type": "meta", "app_id": "...", "title": "...", "command": "..."} + - {"type": "closed", "data": "Application exited"} + - {"type": "error", "data": "..."} + + Client sends: + - {"type": "mouse", "action": "click|dblclick|move|scroll", + "x": N, "y": N, "button": 1, "delta": N} + - {"type": "key", "action": "press|release", + "key": "a", "code": "KeyA", "modifiers": ["ctrl"]} + - {"type": "set_fps", "fps": 15} + """ + username = await ws_authenticate(websocket) + await websocket.accept() + + if not app_id: + await websocket.send_json({"type": "error", "data": "app_id required"}) + await websocket.close(code=4000) + return + + app = display_manager.get_app(username, app_id) + if not app: + await websocket.send_json({"type": "error", "data": "App not found"}) + await websocket.close(code=4004) + return + + # Send metadata + await websocket.send_json({ + "type": "meta", + "app_id": app.app_id, + "title": app.title, + "command": app.command, + }) + + # Send last captured frame for instant display on reconnect + if app.last_frame: + await websocket.send_bytes(app.last_frame) + + app.attach_ws(websocket) + log.info("WebSocket attached to app %s (%s) for %s", app_id, app.command, username) + + try: + while True: + msg = await websocket.receive_json() + msg_type = msg.get("type") + + if msg_type in ("mouse", "key"): + await app.send_input(msg) + elif msg_type == "set_fps": + fps = msg.get("fps", 10) + app.target_fps = max(1, min(30, fps)) + except WebSocketDisconnect: + pass + except Exception: + log.exception("Display WS error for %s/%s", username, app_id) + finally: + app.detach_ws(websocket) + log.info("WebSocket detached from app %s for %s", app_id, username) diff --git a/backend/routers/settings.py b/backend/routers/settings.py index 4a1d2ce..f73852c 100644 --- a/backend/routers/settings.py +++ b/backend/routers/settings.py @@ -21,6 +21,7 @@ class ConfigUpdate(BaseModel): ntp_enabled: Optional[bool] = None dock_apps: Optional[list[str]] = None panel_services: Optional[list[str]] = None + gui_apps: Optional[list[dict]] = None session_timeout_minutes: Optional[int] = None stats_interval_seconds: Optional[int] = None diff --git a/frontend/css/apps/display.css b/frontend/css/apps/display.css new file mode 100644 index 0000000..006ea11 --- /dev/null +++ b/frontend/css/apps/display.css @@ -0,0 +1,74 @@ +/* GUI App display — canvas-based window rendering */ +.app-gui-display { + display: flex; + flex-direction: column; + height: 100%; + background: #0a0a0a; +} + +.gui-canvas-wrap { + flex: 1; + display: flex; + align-items: center; + justify-content: center; + overflow: hidden; + position: relative; +} + +.gui-canvas { + max-width: 100%; + max-height: 100%; + cursor: default; + outline: none; + image-rendering: auto; +} + +/* Status overlays */ +.gui-status { + position: absolute; + inset: 0; + display: flex; + flex-direction: column; + align-items: center; + justify-content: center; + gap: 12px; + font-family: var(--font-mono); + font-size: 13px; + color: var(--text-muted); + background: var(--bg-stage); +} + +.gui-status-icon { + font-size: 32px; + opacity: 0.5; +} + +.gui-status-text { + text-align: center; + line-height: 1.6; +} + +.gui-status.connecting .gui-status-icon::after { + content: ''; + display: inline-block; + width: 16px; + height: 16px; + border: 2px solid var(--text-muted); + border-top-color: var(--accent); + border-radius: 50%; + animation: gui-spin 0.8s linear infinite; + margin-left: 8px; + vertical-align: middle; +} + +@keyframes gui-spin { + to { transform: rotate(360deg); } +} + +.gui-status.exited { + color: var(--status-red); +} + +.gui-status.exited .gui-status-icon { + opacity: 0.7; +} diff --git a/frontend/desktop.html b/frontend/desktop.html index c7fadeb..d15bec5 100644 --- a/frontend/desktop.html +++ b/frontend/desktop.html @@ -23,6 +23,7 @@ +
@@ -207,6 +208,7 @@ +