From a73b515258eba8217c443510753627c6d0bc57d4 Mon Sep 17 00:00:00 2001 From: roberts Date: Sat, 14 Mar 2026 23:09:15 -0500 Subject: [PATCH] Add native GUI app support via per-window frame streaming Each configured GUI app (e.g. Nextcloud) gets its own dock icon and opens as a regular Atlus tab. Under the hood: Xvfb virtual display, ImageMagick captures individual window pixmaps as JPEG, streams over WebSocket to a canvas element, with xdotool forwarding mouse/keyboard input back to the X11 window. Apps persist in background when tab is closed, and streaming pauses when no viewers are attached. New files: backend/display.py (DisplayManager + ManagedGuiApp), backend/routers/display.py (WebSocket + REST), frontend display.js/css. Config: gui_apps array in settings for registered applications. Co-Authored-By: Claude Opus 4.6 --- backend/config.py | 1 + backend/display.py | 499 ++++++++++++++++++++++++++++++++++ backend/main.py | 6 +- backend/routers/display.py | 161 +++++++++++ backend/routers/settings.py | 1 + frontend/css/apps/display.css | 74 +++++ frontend/desktop.html | 2 + frontend/js/apps/display.js | 329 ++++++++++++++++++++++ install.sh | 4 + 9 files changed, 1076 insertions(+), 1 deletion(-) create mode 100644 backend/display.py create mode 100644 backend/routers/display.py create mode 100644 frontend/css/apps/display.css create mode 100644 frontend/js/apps/display.js diff --git a/backend/config.py b/backend/config.py index 899ce75..2517978 100644 --- a/backend/config.py +++ b/backend/config.py @@ -63,6 +63,7 @@ _DEFAULT_CONFIG: dict = { "terminal", "files", "services", "tasks", "network", "settings" ], "panel_services": [], # systemd unit names to show in panel + "gui_apps": [], # GUI apps: [{id, name, command, icon, args, target_fps}] "asi_bridge": { "cifs_share": "//192.168.10.120/share", "mount_point": "/mnt/asiair", diff --git a/backend/display.py b/backend/display.py new file mode 100644 index 0000000..8c59a96 --- /dev/null +++ b/backend/display.py @@ -0,0 +1,499 @@ +"""Atlus — GUI application display manager. + +Manages a virtual X11 display (Xvfb) and per-window frame capture +for streaming native GUI applications to the browser via WebSocket. +Each GUI app window is captured independently and streamed as JPEG frames. +""" + +import asyncio +import logging +import os +import re +import shutil +import signal +import time +import uuid +from dataclasses import dataclass, field +from typing import Optional + +log = logging.getLogger("atlus.display") + +# --------------------------------------------------------------------------- +# Binary discovery (systemd strips PATH) +# --------------------------------------------------------------------------- + +_SEARCH_PATHS = ("/usr/bin", "/usr/local/bin", "/bin", "/usr/sbin", "/sbin") + + +def _find_bin(name: str) -> Optional[str]: + found = shutil.which(name) + if found: + return found + for d in _SEARCH_PATHS: + p = os.path.join(d, name) + if os.path.isfile(p) and os.access(p, os.X_OK): + return p + return None + + +XVFB_BIN = _find_bin("Xvfb") +XDOTOOL_BIN = _find_bin("xdotool") +IMPORT_BIN = _find_bin("import") # ImageMagick + +HAS_DISPLAY_DEPS = all((XVFB_BIN, XDOTOOL_BIN, IMPORT_BIN)) + +# Allowed command pattern — alphanumeric + hyphens only +_SAFE_CMD = re.compile(r"^[a-zA-Z0-9][a-zA-Z0-9._\-]*$") + +# Key translation: DOM key names → xdotool key names +_KEY_MAP = { + "Control": "ctrl", + "Shift": "shift", + "Alt": "alt", + "Meta": "super", + "Enter": "Return", + "Backspace": "BackSpace", + "Delete": "Delete", + "Escape": "Escape", + "Tab": "Tab", + "ArrowUp": "Up", + "ArrowDown": "Down", + "ArrowLeft": "Left", + "ArrowRight": "Right", + "Home": "Home", + "End": "End", + "PageUp": "Prior", + "PageDown": "Next", + "Insert": "Insert", + " ": "space", + "F1": "F1", "F2": "F2", "F3": "F3", "F4": "F4", + "F5": "F5", "F6": "F6", "F7": "F7", "F8": "F8", + "F9": "F9", "F10": "F10", "F11": "F11", "F12": "F12", +} + + +# --------------------------------------------------------------------------- +# ManagedGuiApp — a GUI process with per-window frame capture +# --------------------------------------------------------------------------- + +@dataclass +class ManagedGuiApp: + """A GUI application running on a virtual X display.""" + + app_id: str + command: str + title: str + display_num: int + process: Optional[asyncio.subprocess.Process] = None + window_id: Optional[int] = None + target_fps: int = 10 + created_at: float = field(default_factory=time.time) + last_frame: Optional[bytes] = field(default=None, repr=False) + _capture_task: Optional[asyncio.Task] = field(default=None, repr=False) + _websockets: list = field(default_factory=list, repr=False) + _streaming: bool = field(default=False, repr=False) + + @property + def alive(self) -> bool: + return self.process is not None and self.process.returncode is None + + def _display_env(self) -> dict: + env = {**os.environ, "DISPLAY": f":{self.display_num}"} + # Ensure PATH has common binary dirs + path = env.get("PATH", "") + for p in _SEARCH_PATHS: + if p not in path: + path = p + ":" + path + env["PATH"] = path + return env + + # ---- WebSocket fan-out ---- + + def attach_ws(self, ws): + if ws not in self._websockets: + self._websockets.append(ws) + self._streaming = True + + def detach_ws(self, ws): + try: + self._websockets.remove(ws) + except ValueError: + pass + if not self._websockets: + self._streaming = False + + # ---- Frame capture ---- + + def start_capture(self): + if self._capture_task is None or self._capture_task.done(): + self._capture_task = asyncio.create_task(self._capture_loop()) + + async def _capture_loop(self): + """Background: capture window pixmap → JPEG → fan-out.""" + # Wait for window to appear + for attempt in range(20): + if not self.alive: + return + await self._discover_window() + if self.window_id: + break + await asyncio.sleep(0.5 * (1 + attempt * 0.2)) + + if not self.window_id: + log.warning("No window found for app %s (%s)", self.app_id, self.command) + # Notify any attached viewers + for ws in list(self._websockets): + try: + await ws.send_json({"type": "error", "data": "No window found for application"}) + except Exception: + pass + return + + log.info("Capture loop started for app %s window %d", self.app_id, self.window_id) + + while self.alive: + if not self._streaming or not self._websockets: + await asyncio.sleep(0.5) + continue + try: + frame = await self._capture_frame() + if frame: + self.last_frame = frame + dead = [] + for ws in self._websockets: + try: + await ws.send_bytes(frame) + except Exception: + dead.append(ws) + for ws in dead: + self.detach_ws(ws) + except Exception: + log.exception("Capture error for %s", self.app_id) + break + await asyncio.sleep(1.0 / self.target_fps) + + # Process exited — notify viewers + for ws in list(self._websockets): + try: + await ws.send_json({"type": "closed", "data": "Application exited"}) + except Exception: + pass + log.info("Capture loop ended for app %s", self.app_id) + + async def _capture_frame(self) -> Optional[bytes]: + """Capture window as JPEG using ImageMagick import.""" + if not self.window_id or not IMPORT_BIN: + return None + try: + proc = await asyncio.create_subprocess_exec( + IMPORT_BIN, "-window", str(self.window_id), + "-quality", "60", "jpeg:-", + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + env=self._display_env(), + ) + stdout, stderr = await asyncio.wait_for(proc.communicate(), timeout=5) + if proc.returncode == 0 and stdout: + return stdout + return None + except asyncio.TimeoutError: + return None + except Exception: + return None + + async def _discover_window(self): + """Find X11 window ID for this app's process.""" + if not XDOTOOL_BIN or not self.process: + return + try: + proc = await asyncio.create_subprocess_exec( + XDOTOOL_BIN, "search", "--pid", str(self.process.pid), + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + env=self._display_env(), + ) + stdout, _ = await asyncio.wait_for(proc.communicate(), timeout=5) + lines = stdout.decode().strip().splitlines() + if lines: + self.window_id = int(lines[0]) + log.debug("Discovered window %d for app %s (pid %d)", + self.window_id, self.app_id, self.process.pid) + except Exception: + pass + + # ---- Input forwarding ---- + + async def send_input(self, msg: dict): + """Forward mouse/keyboard input to the X11 window.""" + if not self.window_id or not XDOTOOL_BIN: + return + env = self._display_env() + msg_type = msg.get("type") + + try: + if msg_type == "mouse": + await self._handle_mouse(msg, env) + elif msg_type == "key": + await self._handle_key(msg, env) + except Exception: + log.debug("Input forwarding error for %s", self.app_id, exc_info=True) + + async def _handle_mouse(self, msg: dict, env: dict): + action = msg.get("action", "") + x, y = str(msg.get("x", 0)), str(msg.get("y", 0)) + wid = str(self.window_id) + + if action == "click": + btn = str(msg.get("button", 1)) + await self._xdotool( + "mousemove", "--window", wid, x, y, + "click", "--window", wid, btn, + env=env, + ) + elif action == "dblclick": + btn = str(msg.get("button", 1)) + await self._xdotool( + "mousemove", "--window", wid, x, y, + "click", "--window", wid, "--repeat", "2", btn, + env=env, + ) + elif action == "move": + await self._xdotool( + "mousemove", "--window", wid, x, y, + env=env, + ) + elif action == "scroll": + delta = msg.get("delta", 0) + btn = "4" if delta < 0 else "5" # X11: 4=up, 5=down + await self._xdotool( + "mousemove", "--window", wid, x, y, + "click", "--window", wid, btn, + env=env, + ) + + async def _handle_key(self, msg: dict, env: dict): + action = msg.get("action", "press") + key = msg.get("key", "") + modifiers = msg.get("modifiers", []) + + # Translate key name + xkey = _KEY_MAP.get(key, key) + + # Skip standalone modifier key events + if xkey in ("ctrl", "shift", "alt", "super"): + return + + # Build modifier prefix + mod_parts = [] + for m in modifiers: + xmod = _KEY_MAP.get(m, m.lower()) + if xmod not in ("ctrl", "shift", "alt", "super"): + continue + mod_parts.append(xmod) + + if mod_parts: + xkey = "+".join(mod_parts) + "+" + xkey + + wid = str(self.window_id) + if action == "press": + await self._xdotool("key", "--window", wid, xkey, env=env) + # release events handled implicitly by xdotool key + + async def _xdotool(self, *args, env=None): + proc = await asyncio.create_subprocess_exec( + XDOTOOL_BIN, *args, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + env=env, + ) + await asyncio.wait_for(proc.communicate(), timeout=5) + + # ---- Lifecycle ---- + + def kill(self): + if self._capture_task and not self._capture_task.done(): + self._capture_task.cancel() + if self.alive: + try: + self.process.terminate() + except Exception: + pass + + def to_dict(self) -> dict: + return { + "app_id": self.app_id, + "command": self.command, + "title": self.title, + "alive": self.alive, + "pid": self.process.pid if self.process else None, + "window_id": self.window_id, + "streaming": self._streaming, + "viewers": len(self._websockets), + "created_at": self.created_at, + } + + +# --------------------------------------------------------------------------- +# Display Manager — manages Xvfb displays and GUI apps +# --------------------------------------------------------------------------- + +class DisplayManager: + """Manages virtual X displays and GUI application lifecycle.""" + + def __init__(self): + self._xvfb_procs: dict[str, asyncio.subprocess.Process] = {} + self._display_nums: dict[str, int] = {} + self._apps: dict[str, dict[str, ManagedGuiApp]] = {} + self._next_display = 99 + + def _require_deps(self): + if not HAS_DISPLAY_DEPS: + missing = [] + if not XVFB_BIN: + missing.append("Xvfb") + if not XDOTOOL_BIN: + missing.append("xdotool") + if not IMPORT_BIN: + missing.append("import (ImageMagick)") + raise RuntimeError(f"Missing display dependencies: {', '.join(missing)}") + + async def get_or_create_display(self, username: str) -> int: + """Start Xvfb if needed, return display number.""" + self._require_deps() + + if username in self._display_nums: + # Check if Xvfb is still running + xvfb = self._xvfb_procs.get(username) + if xvfb and xvfb.returncode is None: + return self._display_nums[username] + # Xvfb died — restart + log.warning("Xvfb died for %s, restarting", username) + + display_num = self._next_display + self._next_display += 1 + + log.info("Starting Xvfb :%d for %s", display_num, username) + xvfb = await asyncio.create_subprocess_exec( + XVFB_BIN, f":{display_num}", + "-screen", "0", "1280x1024x24", + "-ac", # disable access control + "-nolisten", "tcp", + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + + # Wait briefly for Xvfb to start + await asyncio.sleep(0.5) + if xvfb.returncode is not None: + raise RuntimeError(f"Xvfb failed to start (rc={xvfb.returncode})") + + self._xvfb_procs[username] = xvfb + self._display_nums[username] = display_num + log.info("Xvfb :%d started for %s (pid %d)", display_num, username, xvfb.pid) + return display_num + + async def launch_app( + self, username: str, command: str, title: str = "", + args: list[str] | None = None, target_fps: int = 10, + ) -> ManagedGuiApp: + """Launch a GUI application on the user's virtual display.""" + # Validate command + if not _SAFE_CMD.match(command): + raise ValueError(f"Invalid command name: {command}") + + # Find the binary + cmd_bin = _find_bin(command) + if not cmd_bin: + raise FileNotFoundError(f"Command not found: {command}") + + display_num = await self.get_or_create_display(username) + + env = { + **os.environ, + "DISPLAY": f":{display_num}", + "HOME": os.path.expanduser("~"), + } + # Ensure PATH + path = env.get("PATH", "") + for p in _SEARCH_PATHS: + if p not in path: + path = p + ":" + path + env["PATH"] = path + + cmd = [cmd_bin] + (args or []) + log.info("Launching GUI app: %s (display :%d, user %s)", " ".join(cmd), display_num, username) + + process = await asyncio.create_subprocess_exec( + *cmd, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + env=env, + ) + + app_id = str(uuid.uuid4())[:8] + app = ManagedGuiApp( + app_id=app_id, + command=command, + title=title or command, + display_num=display_num, + process=process, + target_fps=target_fps, + ) + app.start_capture() + + self._apps.setdefault(username, {})[app_id] = app + log.info("GUI app %s launched: %s (pid %d)", app_id, command, process.pid) + return app + + def get_app(self, username: str, app_id: str) -> Optional[ManagedGuiApp]: + return self._apps.get(username, {}).get(app_id) + + def get_app_by_command(self, username: str, command: str) -> Optional[ManagedGuiApp]: + """Find a running app by command name.""" + for app in self._apps.get(username, {}).values(): + if app.command == command and app.alive: + return app + return None + + def close_app(self, username: str, app_id: str) -> bool: + user_apps = self._apps.get(username, {}) + app = user_apps.pop(app_id, None) + if not app: + return False + app.kill() + log.info("Closed GUI app %s (%s) for %s", app_id, app.command, username) + return True + + def list_apps(self, username: str) -> list[dict]: + user_apps = self._apps.get(username, {}) + # Prune dead apps + dead = [aid for aid, a in user_apps.items() if not a.alive] + for aid in dead: + app = user_apps.pop(aid) + app.kill() + return [a.to_dict() for a in user_apps.values()] + + async def shutdown_all(self): + """Kill all apps and Xvfb displays.""" + for username, apps in self._apps.items(): + for app in apps.values(): + app.kill() + self._apps.clear() + + for username, xvfb in self._xvfb_procs.items(): + if xvfb.returncode is None: + try: + xvfb.terminate() + await asyncio.wait_for(xvfb.wait(), timeout=5) + except Exception: + xvfb.kill() + self._xvfb_procs.clear() + self._display_nums.clear() + log.info("All displays shut down") + + +# --------------------------------------------------------------------------- +# Singleton +# --------------------------------------------------------------------------- + +display_manager = DisplayManager() diff --git a/backend/main.py b/backend/main.py index 98a7ca4..6f40a8d 100644 --- a/backend/main.py +++ b/backend/main.py @@ -11,8 +11,9 @@ from pydantic import BaseModel from backend.auth import authenticate_user, create_token, logout from backend.config import FRONTEND_DIR, HOST, PORT -from backend.routers import stats, terminal, files, services, processes, settings, network, packages, updates, session +from backend.routers import stats, terminal, files, services, processes, settings, network, packages, updates, session, display from backend.sessions import manager as session_manager +from backend.display import display_manager from backend.routers.plugins import asi_bridge logging.basicConfig( @@ -55,6 +56,8 @@ async def lifespan(app: FastAPI): # Kill all PTYs on shutdown session_manager.shutdown_all() + # Stop all virtual displays and GUI apps + await display_manager.shutdown_all() log.info("Atlus shutdown complete") @@ -109,6 +112,7 @@ app.include_router(network.router) app.include_router(packages.router) app.include_router(updates.router) app.include_router(session.router) +app.include_router(display.router) app.include_router(asi_bridge.router) diff --git a/backend/routers/display.py b/backend/routers/display.py new file mode 100644 index 0000000..eb2e869 --- /dev/null +++ b/backend/routers/display.py @@ -0,0 +1,161 @@ +"""Display — WebSocket frame streaming + REST app management for GUI apps. + +Pattern mirrors terminal.py: WebSocket attaches to a running GUI app, +receives JPEG frames, sends input events. REST endpoints manage app lifecycle. +""" + +import logging +from typing import Optional + +from fastapi import APIRouter, Depends, HTTPException, Query, WebSocket, WebSocketDisconnect +from pydantic import BaseModel + +from backend.auth import get_current_user, ws_authenticate +from backend.display import display_manager, HAS_DISPLAY_DEPS + +router = APIRouter(prefix="/api/display", tags=["display"]) +log = logging.getLogger("atlus.display.router") + + +# --------------------------------------------------------------------------- +# Guards +# --------------------------------------------------------------------------- + +def _require_deps(): + if not HAS_DISPLAY_DEPS: + raise HTTPException(503, "Display dependencies not installed (Xvfb, xdotool, ImageMagick)") + + +# --------------------------------------------------------------------------- +# REST — app lifecycle +# --------------------------------------------------------------------------- + +class AppLaunchRequest(BaseModel): + command: str + title: str = "" + args: list[str] = [] + target_fps: int = 10 + + +@router.get("/apps") +async def list_apps(user: str = Depends(get_current_user)): + """List running GUI apps for the current user.""" + _require_deps() + return {"apps": display_manager.list_apps(user)} + + +@router.post("/apps") +async def launch_app(req: AppLaunchRequest, user: str = Depends(get_current_user)): + """Launch a GUI application on the user's virtual display.""" + _require_deps() + + # Check if this command is already running + existing = display_manager.get_app_by_command(user, req.command) + if existing: + return existing.to_dict() + + try: + app = await display_manager.launch_app( + user, req.command, req.title, req.args, req.target_fps, + ) + return app.to_dict() + except ValueError as e: + raise HTTPException(400, str(e)) + except FileNotFoundError as e: + raise HTTPException(404, str(e)) + except RuntimeError as e: + raise HTTPException(500, str(e)) + + +@router.delete("/apps/{app_id}") +async def close_app(app_id: str, user: str = Depends(get_current_user)): + """Stop a running GUI application.""" + _require_deps() + if not display_manager.close_app(user, app_id): + raise HTTPException(404, "App not found") + return {"ok": True} + + +@router.get("/status") +async def display_status(user: str = Depends(get_current_user)): + """Check display system availability.""" + return { + "available": HAS_DISPLAY_DEPS, + "apps": display_manager.list_apps(user), + } + + +# --------------------------------------------------------------------------- +# WebSocket — frame streaming + input +# --------------------------------------------------------------------------- + +@router.websocket("/ws") +async def display_ws( + websocket: WebSocket, + app_id: str = Query(default=None), +): + """Attach to a GUI app's window — receive JPEG frames, send input. + + Query params: + - app_id: required — the GUI app to stream + - token: auth token (handled by ws_authenticate) + + Server sends: + - Binary: raw JPEG frame bytes + - {"type": "meta", "app_id": "...", "title": "...", "command": "..."} + - {"type": "closed", "data": "Application exited"} + - {"type": "error", "data": "..."} + + Client sends: + - {"type": "mouse", "action": "click|dblclick|move|scroll", + "x": N, "y": N, "button": 1, "delta": N} + - {"type": "key", "action": "press|release", + "key": "a", "code": "KeyA", "modifiers": ["ctrl"]} + - {"type": "set_fps", "fps": 15} + """ + username = await ws_authenticate(websocket) + await websocket.accept() + + if not app_id: + await websocket.send_json({"type": "error", "data": "app_id required"}) + await websocket.close(code=4000) + return + + app = display_manager.get_app(username, app_id) + if not app: + await websocket.send_json({"type": "error", "data": "App not found"}) + await websocket.close(code=4004) + return + + # Send metadata + await websocket.send_json({ + "type": "meta", + "app_id": app.app_id, + "title": app.title, + "command": app.command, + }) + + # Send last captured frame for instant display on reconnect + if app.last_frame: + await websocket.send_bytes(app.last_frame) + + app.attach_ws(websocket) + log.info("WebSocket attached to app %s (%s) for %s", app_id, app.command, username) + + try: + while True: + msg = await websocket.receive_json() + msg_type = msg.get("type") + + if msg_type in ("mouse", "key"): + await app.send_input(msg) + elif msg_type == "set_fps": + fps = msg.get("fps", 10) + app.target_fps = max(1, min(30, fps)) + except WebSocketDisconnect: + pass + except Exception: + log.exception("Display WS error for %s/%s", username, app_id) + finally: + app.detach_ws(websocket) + log.info("WebSocket detached from app %s for %s", app_id, username) diff --git a/backend/routers/settings.py b/backend/routers/settings.py index 4a1d2ce..f73852c 100644 --- a/backend/routers/settings.py +++ b/backend/routers/settings.py @@ -21,6 +21,7 @@ class ConfigUpdate(BaseModel): ntp_enabled: Optional[bool] = None dock_apps: Optional[list[str]] = None panel_services: Optional[list[str]] = None + gui_apps: Optional[list[dict]] = None session_timeout_minutes: Optional[int] = None stats_interval_seconds: Optional[int] = None diff --git a/frontend/css/apps/display.css b/frontend/css/apps/display.css new file mode 100644 index 0000000..006ea11 --- /dev/null +++ b/frontend/css/apps/display.css @@ -0,0 +1,74 @@ +/* GUI App display — canvas-based window rendering */ +.app-gui-display { + display: flex; + flex-direction: column; + height: 100%; + background: #0a0a0a; +} + +.gui-canvas-wrap { + flex: 1; + display: flex; + align-items: center; + justify-content: center; + overflow: hidden; + position: relative; +} + +.gui-canvas { + max-width: 100%; + max-height: 100%; + cursor: default; + outline: none; + image-rendering: auto; +} + +/* Status overlays */ +.gui-status { + position: absolute; + inset: 0; + display: flex; + flex-direction: column; + align-items: center; + justify-content: center; + gap: 12px; + font-family: var(--font-mono); + font-size: 13px; + color: var(--text-muted); + background: var(--bg-stage); +} + +.gui-status-icon { + font-size: 32px; + opacity: 0.5; +} + +.gui-status-text { + text-align: center; + line-height: 1.6; +} + +.gui-status.connecting .gui-status-icon::after { + content: ''; + display: inline-block; + width: 16px; + height: 16px; + border: 2px solid var(--text-muted); + border-top-color: var(--accent); + border-radius: 50%; + animation: gui-spin 0.8s linear infinite; + margin-left: 8px; + vertical-align: middle; +} + +@keyframes gui-spin { + to { transform: rotate(360deg); } +} + +.gui-status.exited { + color: var(--status-red); +} + +.gui-status.exited .gui-status-icon { + opacity: 0.7; +} diff --git a/frontend/desktop.html b/frontend/desktop.html index c7fadeb..d15bec5 100644 --- a/frontend/desktop.html +++ b/frontend/desktop.html @@ -23,6 +23,7 @@ + @@ -207,6 +208,7 @@ + diff --git a/frontend/js/apps/display.js b/frontend/js/apps/display.js new file mode 100644 index 0000000..496c307 --- /dev/null +++ b/frontend/js/apps/display.js @@ -0,0 +1,329 @@ +/* Atlus — GUI App Display module. + * + * Provides per-window frame streaming for native GUI applications. + * Each configured gui_app registers as its own Atlus app (like terminal, files). + * Canvas receives JPEG frames via WebSocket, input forwarded back. + */ +(function () { + 'use strict'; + + // Per-app state: keyed by gui app id (e.g. "nextcloud") + const appState = {}; + + // ---- Shared canvas/WS infrastructure ---- + + function createGuiApp(guiConfig) { + const appId = 'gui-' + guiConfig.id; + + Atlus.registerApp(appId, { + title: guiConfig.name || guiConfig.command, + + init(container) { + container.classList.add('app-gui-display'); + + const state = { + config: guiConfig, + container: container, + canvas: null, + ctx: null, + ws: null, + serverAppId: null, // backend app_id + status: 'connecting', + }; + appState[guiConfig.id] = state; + + // Status overlay + const statusEl = document.createElement('div'); + statusEl.className = 'gui-status connecting'; + statusEl.innerHTML = ` +
🖥
+
Launching ${guiConfig.name || guiConfig.command}…
+ `; + container.appendChild(statusEl); + state.statusEl = statusEl; + + // Canvas wrapper + const wrap = document.createElement('div'); + wrap.className = 'gui-canvas-wrap'; + wrap.style.display = 'none'; + + const canvas = document.createElement('canvas'); + canvas.className = 'gui-canvas'; + canvas.width = 800; + canvas.height = 600; + canvas.tabIndex = 0; + wrap.appendChild(canvas); + container.appendChild(wrap); + + state.wrap = wrap; + state.canvas = canvas; + state.ctx = canvas.getContext('2d'); + + // Bind input handlers + bindInput(state); + + // Launch the app on the server, then connect WS + launchAndConnect(state); + }, + + destroy() { + const state = appState[guiConfig.id]; + if (!state) return; + if (state.ws) { + try { state.ws.close(); } catch (e) {} + state.ws = null; + } + // Don't kill the backend app — it runs in background + delete appState[guiConfig.id]; + }, + + onFocus() { + const state = appState[guiConfig.id]; + if (state && state.canvas) { + state.canvas.focus(); + } + }, + }); + } + + async function launchAndConnect(state) { + try { + // Launch or get existing app + const res = await Atlus.apiFetch('/api/display/apps', { + method: 'POST', + body: { + command: state.config.command, + title: state.config.name || state.config.command, + args: state.config.args || [], + target_fps: state.config.target_fps || 10, + }, + }); + + if (!res || !res.ok) { + const err = res ? await res.json().catch(() => ({})) : {}; + showStatus(state, 'error', err.detail || 'Failed to launch application'); + return; + } + + const data = await res.json(); + state.serverAppId = data.app_id; + + // Connect WebSocket + connectWs(state); + } catch (e) { + showStatus(state, 'error', 'Failed to launch: ' + e.message); + } + } + + function connectWs(state) { + if (!state.serverAppId) return; + + const url = Atlus.wsUrl(`/api/display/ws?app_id=${state.serverAppId}`); + const ws = new WebSocket(url); + ws.binaryType = 'arraybuffer'; + state.ws = ws; + + ws.onopen = () => { + showStatus(state, 'connecting', 'Waiting for window…'); + }; + + ws.onmessage = (e) => { + if (e.data instanceof ArrayBuffer) { + // Binary = JPEG frame + renderFrame(state, e.data); + } else { + // JSON message + try { + const msg = JSON.parse(e.data); + handleMessage(state, msg); + } catch (err) {} + } + }; + + ws.onclose = () => { + // Don't auto-reconnect — the app tab will re-init if reopened + }; + + ws.onerror = () => { + showStatus(state, 'error', 'Connection lost'); + }; + } + + function renderFrame(state, buffer) { + // Show canvas, hide status + if (state.status !== 'streaming') { + state.status = 'streaming'; + state.statusEl.style.display = 'none'; + state.wrap.style.display = 'flex'; + state.canvas.focus(); + } + + const blob = new Blob([buffer], { type: 'image/jpeg' }); + const img = new Image(); + img.onload = () => { + if (state.canvas.width !== img.width || state.canvas.height !== img.height) { + state.canvas.width = img.width; + state.canvas.height = img.height; + } + state.ctx.drawImage(img, 0, 0); + URL.revokeObjectURL(img.src); + }; + img.src = URL.createObjectURL(blob); + } + + function handleMessage(state, msg) { + if (msg.type === 'closed') { + showStatus(state, 'exited', msg.data || 'Application exited'); + } else if (msg.type === 'error') { + showStatus(state, 'error', msg.data || 'Error'); + } else if (msg.type === 'meta') { + // Could update title etc. + } + } + + function showStatus(state, type, text) { + state.status = type; + state.wrap.style.display = 'none'; + state.statusEl.style.display = 'flex'; + state.statusEl.className = 'gui-status ' + type; + + const icons = { connecting: '🖥', error: '⚠', exited: '✖' }; + state.statusEl.innerHTML = ` +
${icons[type] || '🖥'}
+
${text}
+ `; + } + + // ---- Input forwarding ---- + + function bindInput(state) { + const canvas = state.canvas; + + function scaleCoords(e) { + const rect = canvas.getBoundingClientRect(); + const scaleX = canvas.width / rect.width; + const scaleY = canvas.height / rect.height; + return { + x: Math.round((e.clientX - rect.left) * scaleX), + y: Math.round((e.clientY - rect.top) * scaleY), + }; + } + + function send(msg) { + if (state.ws && state.ws.readyState === WebSocket.OPEN) { + state.ws.send(JSON.stringify(msg)); + } + } + + function getModifiers(e) { + const m = []; + if (e.ctrlKey) m.push('Control'); + if (e.shiftKey) m.push('Shift'); + if (e.altKey) m.push('Alt'); + if (e.metaKey) m.push('Meta'); + return m; + } + + // Mouse events + canvas.addEventListener('mousedown', (e) => { + canvas.focus(); + const coords = scaleCoords(e); + send({ type: 'mouse', action: 'click', ...coords, button: e.button + 1 }); + }); + + canvas.addEventListener('dblclick', (e) => { + const coords = scaleCoords(e); + send({ type: 'mouse', action: 'dblclick', ...coords, button: e.button + 1 }); + }); + + canvas.addEventListener('mousemove', (e) => { + // Throttle mousemove to ~30fps + if (state._lastMove && Date.now() - state._lastMove < 33) return; + state._lastMove = Date.now(); + const coords = scaleCoords(e); + send({ type: 'mouse', action: 'move', ...coords }); + }); + + canvas.addEventListener('wheel', (e) => { + e.preventDefault(); + const coords = scaleCoords(e); + send({ type: 'mouse', action: 'scroll', ...coords, delta: e.deltaY > 0 ? 1 : -1 }); + }, { passive: false }); + + canvas.addEventListener('contextmenu', (e) => e.preventDefault()); + + // Keyboard events + canvas.addEventListener('keydown', (e) => { + e.preventDefault(); + e.stopPropagation(); + send({ + type: 'key', action: 'press', + key: e.key, code: e.code, + modifiers: getModifiers(e), + }); + }); + + canvas.addEventListener('keyup', (e) => { + e.preventDefault(); + e.stopPropagation(); + send({ + type: 'key', action: 'release', + key: e.key, code: e.code, + modifiers: getModifiers(e), + }); + }); + } + + // ---- Dynamic dock + app registration ---- + + async function initGuiApps() { + try { + const res = await Atlus.apiFetch('/api/settings'); + if (!res || !res.ok) return; + const cfg = await res.json(); + const guiApps = cfg.gui_apps || []; + + if (guiApps.length === 0) return; + + // Check if display system is available + const statusRes = await Atlus.apiFetch('/api/display/status'); + if (!statusRes || !statusRes.ok) return; + const status = await statusRes.json(); + if (!status.available) return; + + const dockApps = document.querySelector('.dock-apps'); + if (!dockApps) return; + + // Find the separator to insert before it + const separator = dockApps.querySelector('.dock-separator'); + + guiApps.forEach(app => { + if (!app.id || !app.command) return; + + // Register the app module + createGuiApp(app); + + // Create dock button + const btn = document.createElement('button'); + btn.className = 'dock-item'; + btn.dataset.app = 'gui-' + app.id; + btn.innerHTML = ` + ${app.icon || '🖥'} + ${app.name || app.command} + `; + btn.addEventListener('click', () => Atlus.openApp('gui-' + app.id)); + + if (separator) { + dockApps.insertBefore(btn, separator); + } else { + dockApps.appendChild(btn); + } + }); + } catch (e) { + // Display not available — silently skip + } + } + + // Initialize after a brief delay to ensure Atlus core is ready + setTimeout(initGuiApps, 200); +})(); diff --git a/install.sh b/install.sh index 7cd2eae..f9ba294 100755 --- a/install.sh +++ b/install.sh @@ -48,6 +48,10 @@ install_deps() { libpam0g-dev \ git \ cifs-utils \ + xvfb \ + xdotool \ + imagemagick \ + x11-utils \ > /dev/null 2>&1 ok "System dependencies installed." }