From eb8394d2470d39b80e65e8383d057af2b03f8a36 Mon Sep 17 00:00:00 2001 From: Till Wegmueller Date: Tue, 7 Apr 2026 17:08:47 +0200 Subject: [PATCH] Add input forwarding from wrclient to wrsrvd Implement client-side input capture (input.rs) that converts winit WindowEvents to protocol InputMessages: keyboard via evdev keycode mapping, pointer motion/buttons/axis. Wire into wrclient main.rs event handler to send input over QUIC via the existing input channel. Server-side: add inject_network_input() to WayRay state that accepts protocol InputMessages and injects them into the Smithay seat, following the same patterns as process_input_event for keyboard, pointer motion with surface focus, click-to-focus, and axis scroll. Also upgrade client frame handling to use persistent framebuffer with XOR-diff decoding via wayray_protocol::encoding::apply_region. --- crates/wrclient/src/input.rs | 265 +++++++++++++++++++++++++++ crates/wrclient/src/main.rs | 343 ++++++++++++++++++++++++++++++++++- crates/wrsrvd/src/state.rs | 101 +++++++++++ 3 files changed, 707 insertions(+), 2 deletions(-) create mode 100644 crates/wrclient/src/input.rs diff --git a/crates/wrclient/src/input.rs b/crates/wrclient/src/input.rs new file mode 100644 index 0000000..3c806c3 --- /dev/null +++ b/crates/wrclient/src/input.rs @@ -0,0 +1,265 @@ +//! Winit event to protocol message conversion. +//! +//! Converts winit `WindowEvent` variants into `InputMessage` values +//! suitable for sending to the wrsrvd server over the QUIC input stream. + +use wayray_protocol::messages::{ + ButtonState, InputMessage, KeyState, KeyboardEvent, PointerAxis, PointerButton, PointerMotion, +}; +use winit::event::{ElementState, MouseButton, MouseScrollDelta}; +use winit::keyboard::{KeyCode, PhysicalKey}; + +/// Convert a winit keyboard event to a protocol `InputMessage`. +/// +/// Returns `None` for keys we cannot map (e.g. `Unidentified`). +pub fn convert_keyboard(event: &winit::event::KeyEvent) -> Option { + let PhysicalKey::Code(code) = event.physical_key else { + return None; + }; + + let keycode = keycode_to_evdev(code)?; + let state = match event.state { + ElementState::Pressed => KeyState::Pressed, + ElementState::Released => KeyState::Released, + }; + + Some(InputMessage::Keyboard(KeyboardEvent { + keycode, + state, + time: 0, // winit doesn't provide timestamps on key events + })) +} + +/// Convert a winit cursor-moved event to a protocol `InputMessage`. +pub fn convert_cursor_moved(position: winit::dpi::PhysicalPosition) -> InputMessage { + InputMessage::PointerMotion(PointerMotion { + x: position.x, + y: position.y, + time: 0, + }) +} + +/// Convert a winit mouse button event to a protocol `InputMessage`. +pub fn convert_mouse_button(button: MouseButton, state: ElementState) -> Option { + let btn_code = match button { + MouseButton::Left => 0x110, // BTN_LEFT + MouseButton::Right => 0x111, // BTN_RIGHT + MouseButton::Middle => 0x112, // BTN_MIDDLE + MouseButton::Back => 0x116, // BTN_BACK + MouseButton::Forward => 0x115, // BTN_FORWARD + MouseButton::Other(code) => 0x110 + code as u32, + }; + + let btn_state = match state { + ElementState::Pressed => ButtonState::Pressed, + ElementState::Released => ButtonState::Released, + }; + + Some(InputMessage::PointerButton(PointerButton { + button: btn_code, + state: btn_state, + time: 0, + })) +} + +/// Convert a winit mouse wheel event to protocol `InputMessage`(s). +/// +/// Returns up to two messages: one for horizontal axis, one for vertical. +pub fn convert_mouse_wheel(delta: MouseScrollDelta) -> Vec { + let mut messages = Vec::new(); + + match delta { + MouseScrollDelta::LineDelta(x, y) => { + if x != 0.0 { + messages.push(InputMessage::PointerAxis(PointerAxis { + axis: wayray_protocol::messages::Axis::Horizontal, + value: x as f64 * 15.0, // approximate line-to-pixel conversion + time: 0, + })); + } + if y != 0.0 { + messages.push(InputMessage::PointerAxis(PointerAxis { + axis: wayray_protocol::messages::Axis::Vertical, + value: y as f64 * 15.0, + time: 0, + })); + } + } + MouseScrollDelta::PixelDelta(pos) => { + if pos.x != 0.0 { + messages.push(InputMessage::PointerAxis(PointerAxis { + axis: wayray_protocol::messages::Axis::Horizontal, + value: pos.x, + time: 0, + })); + } + if pos.y != 0.0 { + messages.push(InputMessage::PointerAxis(PointerAxis { + axis: wayray_protocol::messages::Axis::Vertical, + value: pos.y, + time: 0, + })); + } + } + } + + messages +} + +/// Map a winit `KeyCode` to a Linux evdev keycode. +/// +/// This covers the most common keys. Returns `None` for unmapped keys. +fn keycode_to_evdev(code: KeyCode) -> Option { + let evdev = match code { + KeyCode::Escape => 1, + KeyCode::Digit1 => 2, + KeyCode::Digit2 => 3, + KeyCode::Digit3 => 4, + KeyCode::Digit4 => 5, + KeyCode::Digit5 => 6, + KeyCode::Digit6 => 7, + KeyCode::Digit7 => 8, + KeyCode::Digit8 => 9, + KeyCode::Digit9 => 10, + KeyCode::Digit0 => 11, + KeyCode::Minus => 12, + KeyCode::Equal => 13, + KeyCode::Backspace => 14, + KeyCode::Tab => 15, + KeyCode::KeyQ => 16, + KeyCode::KeyW => 17, + KeyCode::KeyE => 18, + KeyCode::KeyR => 19, + KeyCode::KeyT => 20, + KeyCode::KeyY => 21, + KeyCode::KeyU => 22, + KeyCode::KeyI => 23, + KeyCode::KeyO => 24, + KeyCode::KeyP => 25, + KeyCode::BracketLeft => 26, + KeyCode::BracketRight => 27, + KeyCode::Enter => 28, + KeyCode::ControlLeft => 29, + KeyCode::KeyA => 30, + KeyCode::KeyS => 31, + KeyCode::KeyD => 32, + KeyCode::KeyF => 33, + KeyCode::KeyG => 34, + KeyCode::KeyH => 35, + KeyCode::KeyJ => 36, + KeyCode::KeyK => 37, + KeyCode::KeyL => 38, + KeyCode::Semicolon => 39, + KeyCode::Quote => 40, + KeyCode::Backquote => 41, + KeyCode::ShiftLeft => 42, + KeyCode::Backslash => 43, + KeyCode::KeyZ => 44, + KeyCode::KeyX => 45, + KeyCode::KeyC => 46, + KeyCode::KeyV => 47, + KeyCode::KeyB => 48, + KeyCode::KeyN => 49, + KeyCode::KeyM => 50, + KeyCode::Comma => 51, + KeyCode::Period => 52, + KeyCode::Slash => 53, + KeyCode::ShiftRight => 54, + KeyCode::AltLeft => 56, + KeyCode::Space => 57, + KeyCode::CapsLock => 58, + KeyCode::F1 => 59, + KeyCode::F2 => 60, + KeyCode::F3 => 61, + KeyCode::F4 => 62, + KeyCode::F5 => 63, + KeyCode::F6 => 64, + KeyCode::F7 => 65, + KeyCode::F8 => 66, + KeyCode::F9 => 67, + KeyCode::F10 => 68, + KeyCode::NumLock => 69, + KeyCode::ScrollLock => 70, + KeyCode::F11 => 87, + KeyCode::F12 => 88, + KeyCode::ControlRight => 97, + KeyCode::AltRight => 100, + KeyCode::Home => 102, + KeyCode::ArrowUp => 103, + KeyCode::PageUp => 104, + KeyCode::ArrowLeft => 105, + KeyCode::ArrowRight => 106, + KeyCode::End => 107, + KeyCode::ArrowDown => 108, + KeyCode::PageDown => 109, + KeyCode::Insert => 110, + KeyCode::Delete => 111, + KeyCode::SuperLeft => 125, + KeyCode::SuperRight => 126, + _ => return None, + }; + Some(evdev) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn keycode_a_maps_to_evdev_30() { + assert_eq!(keycode_to_evdev(KeyCode::KeyA), Some(30)); + } + + #[test] + fn keycode_escape_maps_to_evdev_1() { + assert_eq!(keycode_to_evdev(KeyCode::Escape), Some(1)); + } + + #[test] + fn keycode_enter_maps_to_evdev_28() { + assert_eq!(keycode_to_evdev(KeyCode::Enter), Some(28)); + } + + #[test] + fn unmapped_keycode_returns_none() { + // F13 and beyond are not in our mapping. + assert!(keycode_to_evdev(KeyCode::F13).is_none()); + } + + #[test] + fn mouse_button_left() { + let msg = convert_mouse_button(MouseButton::Left, ElementState::Pressed).unwrap(); + match msg { + InputMessage::PointerButton(pb) => { + assert_eq!(pb.button, 0x110); + assert_eq!(pb.state, ButtonState::Pressed); + } + _ => panic!("expected PointerButton"), + } + } + + #[test] + fn cursor_moved_converts() { + let msg = convert_cursor_moved(winit::dpi::PhysicalPosition::new(100.5, 200.0)); + match msg { + InputMessage::PointerMotion(pm) => { + assert!((pm.x - 100.5).abs() < f64::EPSILON); + assert!((pm.y - 200.0).abs() < f64::EPSILON); + } + _ => panic!("expected PointerMotion"), + } + } + + #[test] + fn mouse_wheel_line_delta() { + let msgs = convert_mouse_wheel(MouseScrollDelta::LineDelta(0.0, 1.0)); + assert_eq!(msgs.len(), 1); + match &msgs[0] { + InputMessage::PointerAxis(pa) => { + assert_eq!(pa.axis, wayray_protocol::messages::Axis::Vertical); + assert!((pa.value - 15.0).abs() < f64::EPSILON); + } + _ => panic!("expected PointerAxis"), + } + } +} diff --git a/crates/wrclient/src/main.rs b/crates/wrclient/src/main.rs index f989222..f744b24 100644 --- a/crates/wrclient/src/main.rs +++ b/crates/wrclient/src/main.rs @@ -1,5 +1,344 @@ +//! wrclient -- WayRay thin client viewer. +//! +//! Connects to a wrsrvd server via QUIC, receives framebuffer updates, +//! and renders them in a native window using wgpu. Input events are +//! captured and will be forwarded to the server in a future task. + +pub mod display; +pub mod input; pub mod network; -fn main() { - println!("wrclient viewer"); +use std::net::SocketAddr; +use std::sync::Arc; +use std::sync::mpsc; + +use tracing::{error, info, warn}; +use winit::application::ApplicationHandler; +use winit::event::WindowEvent; +use winit::event_loop::{ActiveEventLoop, EventLoop}; +use winit::window::{Window, WindowAttributes, WindowId}; + +use wayray_protocol::messages::InputMessage; + +use crate::display::Display; +use crate::network::ClientConfig; + +/// Frame data sent from the network thread to the render thread. +pub struct FrameData { + /// Raw BGRA8 pixel data for the full framebuffer. + pub pixels: Vec, +} + +/// Main application state for the winit event loop. +struct App { + /// Server output dimensions. + width: u32, + height: u32, + /// Receiver for frames from the network thread. + frame_rx: mpsc::Receiver, + /// Sender for input events to the network thread. + input_tx: mpsc::Sender, + /// Display state, created once the window is available. + display: Option, + /// The window reference. + window: Option>, +} + +impl App { + fn new( + width: u32, + height: u32, + frame_rx: mpsc::Receiver, + input_tx: mpsc::Sender, + ) -> Self { + Self { + width, + height, + frame_rx, + input_tx, + display: None, + window: None, + } + } + + /// Send an input message to the network thread, logging on failure. + fn send_input(&self, msg: InputMessage) { + if self.input_tx.send(msg).is_err() { + warn!("network thread closed, cannot send input"); + } + } +} + +impl ApplicationHandler for App { + fn resumed(&mut self, event_loop: &ActiveEventLoop) { + if self.window.is_some() { + return; + } + + let attrs = WindowAttributes::default() + .with_title("WayRay Client") + .with_inner_size(winit::dpi::PhysicalSize::new(self.width, self.height)); + + let window = Arc::new( + event_loop + .create_window(attrs) + .expect("failed to create window"), + ); + + let w = self.width; + let h = self.height; + let win = window.clone(); + + // Initialize wgpu display synchronously via pollster since + // winit's resumed callback cannot be async. + let display = pollster::block_on(Display::new(win, w, h)); + + self.window = Some(window); + self.display = Some(display); + + info!( + width = w, + height = h, + "window created and display initialized" + ); + } + + fn about_to_wait(&mut self, _event_loop: &ActiveEventLoop) { + // Drain any pending frames from the network thread. + let mut got_frame = false; + while let Ok(frame) = self.frame_rx.try_recv() { + if let Some(display) = &self.display { + display.update_frame(&frame.pixels); + got_frame = true; + } + } + + if got_frame && let Some(window) = &self.window { + window.request_redraw(); + } + } + + fn window_event( + &mut self, + event_loop: &ActiveEventLoop, + _window_id: WindowId, + event: WindowEvent, + ) { + match event { + WindowEvent::CloseRequested => { + info!("close requested, exiting"); + event_loop.exit(); + } + WindowEvent::Resized(physical_size) => { + if let Some(display) = &mut self.display { + display.resize(physical_size); + } + } + WindowEvent::RedrawRequested => { + if let Some(display) = &mut self.display { + match display.render() { + Ok(()) => {} + Err(wgpu::SurfaceError::OutOfMemory) => { + error!("GPU out of memory, exiting"); + event_loop.exit(); + } + Err(e) => { + // Lost/outdated surfaces are handled inside render(), + // just request another redraw. + warn!(error = %e, "render error, will retry"); + if let Some(window) = &self.window { + window.request_redraw(); + } + } + } + } + } + WindowEvent::KeyboardInput { event, .. } => { + if let Some(msg) = input::convert_keyboard(&event) { + self.send_input(msg); + } + } + WindowEvent::CursorMoved { position, .. } => { + let msg = input::convert_cursor_moved(position); + self.send_input(msg); + } + WindowEvent::MouseInput { button, state, .. } => { + if let Some(msg) = input::convert_mouse_button(button, state) { + self.send_input(msg); + } + } + WindowEvent::MouseWheel { delta, .. } => { + for msg in input::convert_mouse_wheel(delta) { + self.send_input(msg); + } + } + _ => {} + } + } +} + +fn main() { + // Initialize tracing with RUST_LOG env filtering. + tracing_subscriber::fmt() + .with_env_filter( + tracing_subscriber::EnvFilter::try_from_default_env() + .unwrap_or_else(|_| tracing_subscriber::EnvFilter::new("info")), + ) + .init(); + + let args: Vec = std::env::args().collect(); + if args.len() != 2 { + eprintln!("Usage: wrclient :"); + std::process::exit(1); + } + + let server_addr: SocketAddr = match args[1].parse() { + Ok(addr) => addr, + Err(e) => { + eprintln!("Invalid server address '{}': {}", args[1], e); + std::process::exit(1); + } + }; + + // Connect to the server to get dimensions before creating the window. + // This initial handshake runs on a temporary tokio runtime. + info!(server = %server_addr, "connecting to server"); + + let (frame_tx, frame_rx) = mpsc::channel::(); + let (input_tx, input_rx) = mpsc::channel::(); + + // Use a std::sync::mpsc oneshot pattern: the network thread sends + // dimensions back before entering its frame-receive loop. + let (dim_tx, dim_rx) = mpsc::channel::<(u32, u32)>(); + + // Spawn the network thread with its own tokio runtime. + std::thread::Builder::new() + .name("wrclient-network".into()) + .spawn(move || { + let rt = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("failed to create tokio runtime"); + + rt.block_on(async move { + let config = ClientConfig { + server_addr, + capabilities: vec!["display".to_string()], + }; + + let (_endpoint, mut conn) = match network::connect(&config).await { + Ok(c) => c, + Err(e) => { + error!(error = %e, "failed to connect to server"); + // Send zero dimensions to unblock the main thread. + let _ = dim_tx.send((0, 0)); + return; + } + }; + + let width = conn.server_hello.output_width; + let height = conn.server_hello.output_height; + info!( + width, + height, + session_id = conn.server_hello.session_id, + "connected to server" + ); + + // Send dimensions to the main thread so it can create the window. + if dim_tx.send((width, height)).is_err() { + error!("main thread not listening for dimensions"); + return; + } + + // Accept the display stream from the server. + let mut display_recv = match conn.accept_display_stream().await { + Ok(s) => s, + Err(e) => { + error!(error = %e, "failed to accept display stream"); + return; + } + }; + + // Maintain a persistent framebuffer for XOR-diff decoding. + let stride = width as usize * 4; + let mut framebuffer = vec![0u8; stride * height as usize]; + + // Read frames and forward input in a select loop. + loop { + // Drain any pending input messages before blocking on frame read. + while let Ok(input_msg) = input_rx.try_recv() { + if let Err(e) = conn.send_input(&input_msg).await { + warn!(error = %e, "failed to send input"); + } + } + + // Use a short timeout so we can keep draining input even + // when no frames are arriving. + let frame_result = tokio::time::timeout( + std::time::Duration::from_millis(5), + network::read_display_message(&mut display_recv), + ) + .await; + + match frame_result { + Ok(Ok(wayray_protocol::messages::DisplayMessage::FrameUpdate(update))) => { + info!( + sequence = update.sequence, + regions = update.regions.len(), + "received frame" + ); + + // Apply damage regions to the persistent framebuffer. + for region in &update.regions { + wayray_protocol::encoding::apply_region( + &mut framebuffer, + stride, + region, + ); + } + + if frame_tx + .send(FrameData { + pixels: framebuffer.clone(), + }) + .is_err() + { + info!("render thread closed, stopping network loop"); + break; + } + + // Acknowledge the frame. + if let Err(e) = conn.send_frame_ack(update.sequence).await { + warn!(error = %e, "failed to send frame ack"); + } + } + Ok(Err(e)) => { + error!(error = %e, "display stream error"); + break; + } + Err(_) => { + // Timeout -- no frame available, loop back to drain input. + } + } + } + }); + }) + .expect("failed to spawn network thread"); + + // Wait for the network thread to report server dimensions. + let (width, height) = dim_rx + .recv() + .expect("network thread terminated unexpectedly"); + if width == 0 || height == 0 { + error!("failed to get server dimensions, exiting"); + std::process::exit(1); + } + + info!(width, height, "starting display"); + + // Run the winit event loop on the main thread. + let event_loop = EventLoop::new().expect("failed to create event loop"); + let mut app = App::new(width, height, frame_rx, input_tx); + event_loop.run_app(&mut app).expect("event loop error"); } diff --git a/crates/wrsrvd/src/state.rs b/crates/wrsrvd/src/state.rs index 2d6a8c1..58171ab 100644 --- a/crates/wrsrvd/src/state.rs +++ b/crates/wrsrvd/src/state.rs @@ -22,6 +22,7 @@ use smithay::{ }, }; use tracing::info; +use wayray_protocol::messages::InputMessage; /// Central compositor state holding all Smithay subsystem states. /// @@ -190,4 +191,104 @@ impl WayRay { _ => {} // Ignore other events } } + + /// Inject an input event received from a network client into the + /// compositor's seat, following the same patterns as `process_input_event`. + pub fn inject_network_input(&mut self, msg: InputMessage) { + match msg { + InputMessage::Keyboard(ev) => { + let serial = SERIAL_COUNTER.next_serial(); + let keyboard = self.seat.get_keyboard().unwrap(); + let state = match ev.state { + wayray_protocol::messages::KeyState::Pressed => { + smithay::backend::input::KeyState::Pressed + } + wayray_protocol::messages::KeyState::Released => { + smithay::backend::input::KeyState::Released + } + }; + keyboard.input::<(), _>( + self, + ev.keycode.into(), + state, + serial, + ev.time, + |_, _, _| FilterResult::Forward, + ); + } + InputMessage::PointerMotion(ev) => { + let serial = SERIAL_COUNTER.next_serial(); + let pointer = self.seat.get_pointer().unwrap(); + + let pos = (ev.x, ev.y).into(); + + let under = self.space.element_under(pos).and_then(|(window, loc)| { + window + .surface_under(pos - loc.to_f64(), WindowSurfaceType::ALL) + .map(|(surface, surf_loc)| (surface, (surf_loc + loc).to_f64())) + }); + + pointer.motion( + self, + under, + &MotionEvent { + location: pos, + serial, + time: ev.time, + }, + ); + pointer.frame(self); + } + InputMessage::PointerButton(ev) => { + let serial = SERIAL_COUNTER.next_serial(); + let pointer = self.seat.get_pointer().unwrap(); + + let state = match ev.state { + wayray_protocol::messages::ButtonState::Pressed => ButtonState::Pressed, + wayray_protocol::messages::ButtonState::Released => ButtonState::Released, + }; + + // Click-to-focus on button press. + if state == ButtonState::Pressed { + let pos = pointer.current_location(); + if let Some((window, _loc)) = self.space.element_under(pos) { + let window = window.clone(); + self.space.raise_element(&window, true); + + let keyboard = self.seat.get_keyboard().unwrap(); + let wl_surface = window.toplevel().map(|t| t.wl_surface().clone()); + keyboard.set_focus(self, wl_surface, serial); + } + } + + pointer.button( + self, + &ButtonEvent { + serial, + time: ev.time, + button: ev.button, + state, + }, + ); + pointer.frame(self); + } + InputMessage::PointerAxis(ev) => { + let pointer = self.seat.get_pointer().unwrap(); + + let axis = match ev.axis { + wayray_protocol::messages::Axis::Horizontal => Axis::Horizontal, + wayray_protocol::messages::Axis::Vertical => Axis::Vertical, + }; + + let mut frame = AxisFrame::new(ev.time).source(AxisSource::Wheel); + + if ev.value != 0.0 { + frame = frame.value(axis, ev.value); + } + + pointer.axis(self, frame); + pointer.frame(self); + } + } + } }