atlus/frontend/js/apps/display.js

/* Atlus — GUI App Display module.
 *
 * Provides per-window frame streaming for native GUI applications.
 * Each configured gui_app registers as its own Atlus app (like terminal, files).
 * Canvas receives JPEG frames via WebSocket, input forwarded back.
 */
(function () {
    'use strict';

    // Per-app state: keyed by gui app id (e.g. "nextcloud")
    const appState = {};

    // ---- Shared canvas/WS infrastructure ----

    function createGuiApp(guiConfig) {
        const appId = 'gui-' + guiConfig.id;

        Atlus.registerApp(appId, {
            title: guiConfig.name || guiConfig.command,

            init(container) {
                container.classList.add('app-gui-display');

                const state = {
                    config: guiConfig,
                    container: container,
                    canvas: null,
                    ctx: null,
                    ws: null,
                    serverAppId: null,  // backend app_id
                    status: 'connecting',
                    _lastMove: 0,
                    _inputCount: 0,
                };
                appState[guiConfig.id] = state;

                // Status overlay
                const statusEl = document.createElement('div');
                statusEl.className = 'gui-status connecting';
                statusEl.innerHTML = `
                    <div class="gui-status-icon">🖥</div>
                    <div class="gui-status-text">Launching ${guiConfig.name || guiConfig.command}…</div>
                `;
                container.appendChild(statusEl);
                state.statusEl = statusEl;

                // Canvas wrapper
                const wrap = document.createElement('div');
                wrap.className = 'gui-canvas-wrap';
                wrap.style.display = 'none';

                const canvas = document.createElement('canvas');
                canvas.className = 'gui-canvas';
                canvas.width = 800;
                canvas.height = 600;
                canvas.tabIndex = 0;
                // Ensure canvas is focusable and captures all events
                canvas.style.outline = 'none';
                wrap.appendChild(canvas);
                container.appendChild(wrap);

                state.wrap = wrap;
                state.canvas = canvas;
                state.ctx = canvas.getContext('2d');

                // Bind input handlers
                bindInput(state);

                // Launch the app on the server, then connect WS
                launchAndConnect(state);
            },

            destroy() {
                const state = appState[guiConfig.id];
                if (!state) return;
                if (state.ws) {
                    try { state.ws.close(); } catch (e) {}
                    state.ws = null;
                }
                // Don't kill the backend app — it runs in background
                delete appState[guiConfig.id];
            },

            onFocus() {
                const state = appState[guiConfig.id];
                if (state && state.canvas) {
                    state.canvas.focus();
                }
            },
        });
    }

    async function launchAndConnect(state) {
        try {
            // Launch or get existing app
            const res = await Atlus.apiFetch('/api/display/apps', {
                method: 'POST',
                body: {
                    command: state.config.command,
                    title: state.config.name || state.config.command,
                    args: state.config.args || [],
                    target_fps: state.config.target_fps || 10,
                },
            });

            if (!res || !res.ok) {
                const err = res ? await res.json().catch(() => ({})) : {};
                showStatus(state, 'error', err.detail || 'Failed to launch application');
                return;
            }

            const data = await res.json();
            state.serverAppId = data.app_id;

            // Connect WebSocket
            connectWs(state);
        } catch (e) {
            showStatus(state, 'error', 'Failed to launch: ' + e.message);
        }
    }

    function connectWs(state) {
        if (!state.serverAppId) return;

        const url = Atlus.wsUrl(`/api/display/ws?app_id=${state.serverAppId}`);
        const ws = new WebSocket(url);
        ws.binaryType = 'arraybuffer';
        state.ws = ws;

        ws.onopen = () => {
            console.log('[display] WebSocket connected for', state.config.command);
            showStatus(state, 'connecting', 'Waiting for window…');
        };

        ws.onmessage = (e) => {
            if (e.data instanceof ArrayBuffer) {
                // Binary = JPEG frame
                renderFrame(state, e.data);
            } else {
                // JSON message
                try {
                    const msg = JSON.parse(e.data);
                    console.log('[display] WS message:', msg);
                    handleMessage(state, msg);
                } catch (err) {}
            }
        };

        ws.onclose = (e) => {
            console.log('[display] WebSocket closed:', e.code, e.reason);
        };

        ws.onerror = () => {
            showStatus(state, 'error', 'Connection lost');
        };
    }

    function renderFrame(state, buffer) {
        // Show canvas, hide status
        if (state.status !== 'streaming') {
            state.status = 'streaming';
            state.statusEl.style.display = 'none';
            state.wrap.style.display = 'flex';
            state.canvas.focus();
            console.log('[display] First frame received — canvas visible, input active');
        }

        const blob = new Blob([buffer], { type: 'image/jpeg' });
        const img = new Image();
        img.onload = () => {
            if (state.canvas.width !== img.width || state.canvas.height !== img.height) {
                state.canvas.width = img.width;
                state.canvas.height = img.height;
                console.log('[display] Canvas resized to', img.width, 'x', img.height);
            }
            state.ctx.drawImage(img, 0, 0);
            URL.revokeObjectURL(img.src);
        };
        img.src = URL.createObjectURL(blob);
    }

    function handleMessage(state, msg) {
        if (msg.type === 'closed') {
            showStatus(state, 'exited', msg.data || 'Application exited');
        } else if (msg.type === 'error') {
            showStatus(state, 'error', msg.data || 'Error');
        } else if (msg.type === 'meta') {
            // Could update title etc.
        }
    }

    function showStatus(state, type, text) {
        state.status = type;
        state.wrap.style.display = 'none';
        state.statusEl.style.display = 'flex';
        state.statusEl.className = 'gui-status ' + type;

        const icons = { connecting: '🖥', error: '⚠', exited: '✖' };
        // Escape HTML and preserve newlines for multiline error messages
        const escaped = text.replace(/&/g, '&amp;').replace(/</g, '&lt;').replace(/>/g, '&gt;');
        const formatted = escaped.replace(/\n/g, '<br>');
        state.statusEl.innerHTML = `
            <div class="gui-status-icon">${icons[type] || '🖥'}</div>
            <div class="gui-status-text">${formatted}</div>
        `;
    }

    // ---- Input forwarding ----

    function bindInput(state) {
        const canvas = state.canvas;

        function scaleCoords(e) {
            const rect = canvas.getBoundingClientRect();
            const scaleX = canvas.width / rect.width;
            const scaleY = canvas.height / rect.height;
            return {
                x: Math.round((e.clientX - rect.left) * scaleX),
                y: Math.round((e.clientY - rect.top) * scaleY),
            };
        }

        function send(msg) {
            if (state.ws && state.ws.readyState === WebSocket.OPEN) {
                state.ws.send(JSON.stringify(msg));
                // Log first few events for debugging
                state._inputCount++;
                if (state._inputCount <= 10) {
                    console.log('[display] Input sent:', msg.type, msg.action || msg.key || '');
                } else if (state._inputCount === 11) {
                    console.log('[display] (suppressing further input logs)');
                }
            } else {
                console.warn('[display] Cannot send input — WS not open, state:', state.ws?.readyState);
            }
        }

        function getModifiers(e) {
            const m = [];
            if (e.ctrlKey) m.push('Control');
            if (e.shiftKey) m.push('Shift');
            if (e.altKey) m.push('Alt');
            if (e.metaKey) m.push('Meta');
            return m;
        }

        // Mouse events
        canvas.addEventListener('mousedown', (e) => {
            e.preventDefault();
            e.stopPropagation();
            canvas.focus();
            const coords = scaleCoords(e);
            send({ type: 'mouse', action: 'click', ...coords, button: e.button + 1 });
        });

        canvas.addEventListener('dblclick', (e) => {
            e.preventDefault();
            e.stopPropagation();
            const coords = scaleCoords(e);
            send({ type: 'mouse', action: 'dblclick', ...coords, button: e.button + 1 });
        });

        canvas.addEventListener('mousemove', (e) => {
            // Throttle mousemove to ~30fps
            const now = Date.now();
            if (now - state._lastMove < 33) return;
            state._lastMove = now;
            const coords = scaleCoords(e);
            send({ type: 'mouse', action: 'move', ...coords });
        });

        canvas.addEventListener('wheel', (e) => {
            e.preventDefault();
            e.stopPropagation();
            const coords = scaleCoords(e);
            send({ type: 'mouse', action: 'scroll', ...coords, delta: e.deltaY > 0 ? 1 : -1 });
        }, { passive: false });

        canvas.addEventListener('contextmenu', (e) => {
            e.preventDefault();
            e.stopPropagation();
        });

        // Keyboard events — capture phase to intercept before Atlus shell
        canvas.addEventListener('keydown', (e) => {
            e.preventDefault();
            e.stopPropagation();
            send({
                type: 'key', action: 'press',
                key: e.key, code: e.code,
                modifiers: getModifiers(e),
            });
        }, true);  // capture phase

        canvas.addEventListener('keyup', (e) => {
            e.preventDefault();
            e.stopPropagation();
            send({
                type: 'key', action: 'release',
                key: e.key, code: e.code,
                modifiers: getModifiers(e),
            });
        }, true);  // capture phase
    }

    // ---- App registration (no left dock — apps live in the right panel) ----

    async function initGuiApps() {
        try {
            const res = await Atlus.apiFetch('/api/settings');
            if (!res || !res.ok) return;
            const cfg = await res.json();
            const guiApps = cfg.gui_apps || [];

            if (guiApps.length === 0) return;

            guiApps.forEach(app => {
                if (!app.id || !app.command) return;
                // Register the app module so openApp('gui-xxx') works from the right panel
                createGuiApp(app);
            });
        } catch (e) {
            // Display not available — silently skip
        }
    }

    // Expose for dynamic re-registration when apps are added via Settings
    window._atlusRegisterGuiApp = createGuiApp;

    // Initialize after a brief delay to ensure Atlus core is ready
    setTimeout(initGuiApps, 200);
})();