CloudNebulaProject/wayray
1
0
Fork 0
mirror of https://github.com/CloudNebulaProject/wayray.git synced 2026-04-10 13:10:41 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Add input forwarding from wrclient to wrsrvd

Browse source 
Implement client-side input capture (input.rs) that converts winit
WindowEvents to protocol InputMessages: keyboard via evdev keycode
mapping, pointer motion/buttons/axis. Wire into wrclient main.rs
event handler to send input over QUIC via the existing input channel.

Server-side: add inject_network_input() to WayRay state that accepts
protocol InputMessages and injects them into the Smithay seat, following
the same patterns as process_input_event for keyboard, pointer motion
with surface focus, click-to-focus, and axis scroll.

Also upgrade client frame handling to use persistent framebuffer with
XOR-diff decoding via wayray_protocol::encoding::apply_region.
This commit is contained in:
Till Wegmueller 2026-04-07 17:08:47 +02:00
parent f79a934c2b
commit eb8394d247
3 changed files with 707 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

265
crates/wrclient/src/input.rs Normal file
View file

@ -0,0 +1,265 @@
//! Winit event to protocol message conversion.
//!
//! Converts winit `WindowEvent` variants into `InputMessage` values
//! suitable for sending to the wrsrvd server over the QUIC input stream.
use wayray_protocol::messages::{
ButtonState, InputMessage, KeyState, KeyboardEvent, PointerAxis, PointerButton, PointerMotion,
};
use winit::event::{ElementState, MouseButton, MouseScrollDelta};
use winit::keyboard::{KeyCode, PhysicalKey};
/// Convert a winit keyboard event to a protocol `InputMessage`.
///
/// Returns `None` for keys we cannot map (e.g. `Unidentified`).
pub fn convert_keyboard(event: &winit::event::KeyEvent) -> Option<InputMessage> {
let PhysicalKey::Code(code) = event.physical_key else {
return None;
};
let keycode = keycode_to_evdev(code)?;
let state = match event.state {
ElementState::Pressed => KeyState::Pressed,
ElementState::Released => KeyState::Released,
};
Some(InputMessage::Keyboard(KeyboardEvent {
keycode,
state,
time: 0, // winit doesn't provide timestamps on key events
}))
}
/// Convert a winit cursor-moved event to a protocol `InputMessage`.
pub fn convert_cursor_moved(position: winit::dpi::PhysicalPosition<f64>) -> InputMessage {
InputMessage::PointerMotion(PointerMotion {
x: position.x,
y: position.y,
time: 0,
})
}
/// Convert a winit mouse button event to a protocol `InputMessage`.
pub fn convert_mouse_button(button: MouseButton, state: ElementState) -> Option<InputMessage> {
let btn_code = match button {
MouseButton::Left => 0x110, // BTN_LEFT
MouseButton::Right => 0x111, // BTN_RIGHT
MouseButton::Middle => 0x112, // BTN_MIDDLE
MouseButton::Back => 0x116, // BTN_BACK
MouseButton::Forward => 0x115, // BTN_FORWARD
MouseButton::Other(code) => 0x110 + code as u32,
};
let btn_state = match state {
ElementState::Pressed => ButtonState::Pressed,
ElementState::Released => ButtonState::Released,
};
Some(InputMessage::PointerButton(PointerButton {
button: btn_code,
state: btn_state,
time: 0,
}))
}
/// Convert a winit mouse wheel event to protocol `InputMessage`(s).
///
/// Returns up to two messages: one for horizontal axis, one for vertical.
pub fn convert_mouse_wheel(delta: MouseScrollDelta) -> Vec<InputMessage> {
let mut messages = Vec::new();
match delta {
MouseScrollDelta::LineDelta(x, y) => {
if x != 0.0 {
messages.push(InputMessage::PointerAxis(PointerAxis {
axis: wayray_protocol::messages::Axis::Horizontal,
value: x as f64 * 15.0, // approximate line-to-pixel conversion
time: 0,
}));
}
if y != 0.0 {
messages.push(InputMessage::PointerAxis(PointerAxis {
axis: wayray_protocol::messages::Axis::Vertical,
value: y as f64 * 15.0,
time: 0,
}));
}
}
MouseScrollDelta::PixelDelta(pos) => {
if pos.x != 0.0 {
messages.push(InputMessage::PointerAxis(PointerAxis {
axis: wayray_protocol::messages::Axis::Horizontal,
value: pos.x,
time: 0,
}));
}
if pos.y != 0.0 {
messages.push(InputMessage::PointerAxis(PointerAxis {
axis: wayray_protocol::messages::Axis::Vertical,
value: pos.y,
time: 0,
}));
}
}
}
messages
}
/// Map a winit `KeyCode` to a Linux evdev keycode.
///
/// This covers the most common keys. Returns `None` for unmapped keys.
fn keycode_to_evdev(code: KeyCode) -> Option<u32> {
let evdev = match code {
KeyCode::Escape => 1,
KeyCode::Digit1 => 2,
KeyCode::Digit2 => 3,
KeyCode::Digit3 => 4,
KeyCode::Digit4 => 5,
KeyCode::Digit5 => 6,
KeyCode::Digit6 => 7,
KeyCode::Digit7 => 8,
KeyCode::Digit8 => 9,
KeyCode::Digit9 => 10,
KeyCode::Digit0 => 11,
KeyCode::Minus => 12,
KeyCode::Equal => 13,
KeyCode::Backspace => 14,
KeyCode::Tab => 15,
KeyCode::KeyQ => 16,
KeyCode::KeyW => 17,
KeyCode::KeyE => 18,
KeyCode::KeyR => 19,
KeyCode::KeyT => 20,
KeyCode::KeyY => 21,
KeyCode::KeyU => 22,
KeyCode::KeyI => 23,
KeyCode::KeyO => 24,
KeyCode::KeyP => 25,
KeyCode::BracketLeft => 26,
KeyCode::BracketRight => 27,
KeyCode::Enter => 28,
KeyCode::ControlLeft => 29,
KeyCode::KeyA => 30,
KeyCode::KeyS => 31,
KeyCode::KeyD => 32,
KeyCode::KeyF => 33,
KeyCode::KeyG => 34,
KeyCode::KeyH => 35,
KeyCode::KeyJ => 36,
KeyCode::KeyK => 37,
KeyCode::KeyL => 38,
KeyCode::Semicolon => 39,
KeyCode::Quote => 40,
KeyCode::Backquote => 41,
KeyCode::ShiftLeft => 42,
KeyCode::Backslash => 43,
KeyCode::KeyZ => 44,
KeyCode::KeyX => 45,
KeyCode::KeyC => 46,
KeyCode::KeyV => 47,
KeyCode::KeyB => 48,
KeyCode::KeyN => 49,
KeyCode::KeyM => 50,
KeyCode::Comma => 51,
KeyCode::Period => 52,
KeyCode::Slash => 53,
KeyCode::ShiftRight => 54,
KeyCode::AltLeft => 56,
KeyCode::Space => 57,
KeyCode::CapsLock => 58,
KeyCode::F1 => 59,
KeyCode::F2 => 60,
KeyCode::F3 => 61,
KeyCode::F4 => 62,
KeyCode::F5 => 63,
KeyCode::F6 => 64,
KeyCode::F7 => 65,
KeyCode::F8 => 66,
KeyCode::F9 => 67,
KeyCode::F10 => 68,
KeyCode::NumLock => 69,
KeyCode::ScrollLock => 70,
KeyCode::F11 => 87,
KeyCode::F12 => 88,
KeyCode::ControlRight => 97,
KeyCode::AltRight => 100,
KeyCode::Home => 102,
KeyCode::ArrowUp => 103,
KeyCode::PageUp => 104,
KeyCode::ArrowLeft => 105,
KeyCode::ArrowRight => 106,
KeyCode::End => 107,
KeyCode::ArrowDown => 108,
KeyCode::PageDown => 109,
KeyCode::Insert => 110,
KeyCode::Delete => 111,
KeyCode::SuperLeft => 125,
KeyCode::SuperRight => 126,
_ => return None,
};
Some(evdev)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn keycode_a_maps_to_evdev_30() {
assert_eq!(keycode_to_evdev(KeyCode::KeyA), Some(30));
}
#[test]
fn keycode_escape_maps_to_evdev_1() {
assert_eq!(keycode_to_evdev(KeyCode::Escape), Some(1));
}
#[test]
fn keycode_enter_maps_to_evdev_28() {
assert_eq!(keycode_to_evdev(KeyCode::Enter), Some(28));
}
#[test]
fn unmapped_keycode_returns_none() {
// F13 and beyond are not in our mapping.
assert!(keycode_to_evdev(KeyCode::F13).is_none());
}
#[test]
fn mouse_button_left() {
let msg = convert_mouse_button(MouseButton::Left, ElementState::Pressed).unwrap();
match msg {
InputMessage::PointerButton(pb) => {
assert_eq!(pb.button, 0x110);
assert_eq!(pb.state, ButtonState::Pressed);
}
_ => panic!("expected PointerButton"),
}
}
#[test]
fn cursor_moved_converts() {
let msg = convert_cursor_moved(winit::dpi::PhysicalPosition::new(100.5, 200.0));
match msg {
InputMessage::PointerMotion(pm) => {
assert!((pm.x - 100.5).abs() < f64::EPSILON);
assert!((pm.y - 200.0).abs() < f64::EPSILON);
}
_ => panic!("expected PointerMotion"),
}
}
#[test]
fn mouse_wheel_line_delta() {
let msgs = convert_mouse_wheel(MouseScrollDelta::LineDelta(0.0, 1.0));
assert_eq!(msgs.len(), 1);
match &msgs[0] {
InputMessage::PointerAxis(pa) => {
assert_eq!(pa.axis, wayray_protocol::messages::Axis::Vertical);
assert!((pa.value - 15.0).abs() < f64::EPSILON);
}
_ => panic!("expected PointerAxis"),
}
}
}

343
crates/wrclient/src/main.rs
View file

@ -1,5 +1,344 @@
//! wrclient -- WayRay thin client viewer.
//!
//! Connects to a wrsrvd server via QUIC, receives framebuffer updates,
//! and renders them in a native window using wgpu. Input events are
//! captured and will be forwarded to the server in a future task.
pub mod display;
pub mod input;
pub mod network;
fn main() {
println!("wrclient viewer");
use std::net::SocketAddr;
use std::sync::Arc;
use std::sync::mpsc;
use tracing::{error, info, warn};
use winit::application::ApplicationHandler;
use winit::event::WindowEvent;
use winit::event_loop::{ActiveEventLoop, EventLoop};
use winit::window::{Window, WindowAttributes, WindowId};
use wayray_protocol::messages::InputMessage;
use crate::display::Display;
use crate::network::ClientConfig;
/// Frame data sent from the network thread to the render thread.
pub struct FrameData {
/// Raw BGRA8 pixel data for the full framebuffer.
pub pixels: Vec<u8>,
}
/// Main application state for the winit event loop.
struct App {
/// Server output dimensions.
width: u32,
height: u32,
/// Receiver for frames from the network thread.
frame_rx: mpsc::Receiver<FrameData>,
/// Sender for input events to the network thread.
input_tx: mpsc::Sender<InputMessage>,
/// Display state, created once the window is available.
display: Option<Display>,
/// The window reference.
window: Option<Arc<Window>>,
}
impl App {
fn new(
width: u32,
height: u32,
frame_rx: mpsc::Receiver<FrameData>,
input_tx: mpsc::Sender<InputMessage>,
) -> Self {
Self {
width,
height,
frame_rx,
input_tx,
display: None,
window: None,
}
}
/// Send an input message to the network thread, logging on failure.
fn send_input(&self, msg: InputMessage) {
if self.input_tx.send(msg).is_err() {
warn!("network thread closed, cannot send input");
}
}
}
impl ApplicationHandler for App {
fn resumed(&mut self, event_loop: &ActiveEventLoop) {
if self.window.is_some() {
return;
}
let attrs = WindowAttributes::default()
.with_title("WayRay Client")
.with_inner_size(winit::dpi::PhysicalSize::new(self.width, self.height));
let window = Arc::new(
event_loop
.create_window(attrs)
.expect("failed to create window"),
);
let w = self.width;
let h = self.height;
let win = window.clone();
// Initialize wgpu display synchronously via pollster since
// winit's resumed callback cannot be async.
let display = pollster::block_on(Display::new(win, w, h));
self.window = Some(window);
self.display = Some(display);
info!(
width = w,
height = h,
"window created and display initialized"
);
}
fn about_to_wait(&mut self, _event_loop: &ActiveEventLoop) {
// Drain any pending frames from the network thread.
let mut got_frame = false;
while let Ok(frame) = self.frame_rx.try_recv() {
if let Some(display) = &self.display {
display.update_frame(&frame.pixels);
got_frame = true;
}
}
if got_frame && let Some(window) = &self.window {
window.request_redraw();
}
}
fn window_event(
&mut self,
event_loop: &ActiveEventLoop,
_window_id: WindowId,
event: WindowEvent,
) {
match event {
WindowEvent::CloseRequested => {
info!("close requested, exiting");
event_loop.exit();
}
WindowEvent::Resized(physical_size) => {
if let Some(display) = &mut self.display {
display.resize(physical_size);
}
}
WindowEvent::RedrawRequested => {
if let Some(display) = &mut self.display {
match display.render() {
Ok(()) => {}
Err(wgpu::SurfaceError::OutOfMemory) => {
error!("GPU out of memory, exiting");
event_loop.exit();
}
Err(e) => {
// Lost/outdated surfaces are handled inside render(),
// just request another redraw.
warn!(error = %e, "render error, will retry");
if let Some(window) = &self.window {
window.request_redraw();
}
}
}
}
}
WindowEvent::KeyboardInput { event, .. } => {
if let Some(msg) = input::convert_keyboard(&event) {
self.send_input(msg);
}
}
WindowEvent::CursorMoved { position, .. } => {
let msg = input::convert_cursor_moved(position);
self.send_input(msg);
}
WindowEvent::MouseInput { button, state, .. } => {
if let Some(msg) = input::convert_mouse_button(button, state) {
self.send_input(msg);
}
}
WindowEvent::MouseWheel { delta, .. } => {
for msg in input::convert_mouse_wheel(delta) {
self.send_input(msg);
}
}
_ => {}
}
}
}
fn main() {
// Initialize tracing with RUST_LOG env filtering.
tracing_subscriber::fmt()
.with_env_filter(
tracing_subscriber::EnvFilter::try_from_default_env()
.unwrap_or_else(|_| tracing_subscriber::EnvFilter::new("info")),
)
.init();
let args: Vec<String> = std::env::args().collect();
if args.len() != 2 {
eprintln!("Usage: wrclient <host>:<port>");
std::process::exit(1);
}
let server_addr: SocketAddr = match args[1].parse() {
Ok(addr) => addr,
Err(e) => {
eprintln!("Invalid server address '{}': {}", args[1], e);
std::process::exit(1);
}
};
// Connect to the server to get dimensions before creating the window.
// This initial handshake runs on a temporary tokio runtime.
info!(server = %server_addr, "connecting to server");
let (frame_tx, frame_rx) = mpsc::channel::<FrameData>();
let (input_tx, input_rx) = mpsc::channel::<InputMessage>();
// Use a std::sync::mpsc oneshot pattern: the network thread sends
// dimensions back before entering its frame-receive loop.
let (dim_tx, dim_rx) = mpsc::channel::<(u32, u32)>();
// Spawn the network thread with its own tokio runtime.
std::thread::Builder::new()
.name("wrclient-network".into())
.spawn(move || {
let rt = tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
.expect("failed to create tokio runtime");
rt.block_on(async move {
let config = ClientConfig {
server_addr,
capabilities: vec!["display".to_string()],
};
let (_endpoint, mut conn) = match network::connect(&config).await {
Ok(c) => c,
Err(e) => {
error!(error = %e, "failed to connect to server");
// Send zero dimensions to unblock the main thread.
let _ = dim_tx.send((0, 0));
return;
}
};
let width = conn.server_hello.output_width;
let height = conn.server_hello.output_height;
info!(
width,
height,
session_id = conn.server_hello.session_id,
"connected to server"
);
// Send dimensions to the main thread so it can create the window.
if dim_tx.send((width, height)).is_err() {
error!("main thread not listening for dimensions");
return;
}
// Accept the display stream from the server.
let mut display_recv = match conn.accept_display_stream().await {
Ok(s) => s,
Err(e) => {
error!(error = %e, "failed to accept display stream");
return;
}
};
// Maintain a persistent framebuffer for XOR-diff decoding.
let stride = width as usize * 4;
let mut framebuffer = vec![0u8; stride * height as usize];
// Read frames and forward input in a select loop.
loop {
// Drain any pending input messages before blocking on frame read.
while let Ok(input_msg) = input_rx.try_recv() {
if let Err(e) = conn.send_input(&input_msg).await {
warn!(error = %e, "failed to send input");
}
}
// Use a short timeout so we can keep draining input even
// when no frames are arriving.
let frame_result = tokio::time::timeout(
std::time::Duration::from_millis(5),
network::read_display_message(&mut display_recv),
)
.await;
match frame_result {
Ok(Ok(wayray_protocol::messages::DisplayMessage::FrameUpdate(update))) => {
info!(
sequence = update.sequence,
regions = update.regions.len(),
"received frame"
);
// Apply damage regions to the persistent framebuffer.
for region in &update.regions {
wayray_protocol::encoding::apply_region(
&mut framebuffer,
stride,
region,
);
}
if frame_tx
.send(FrameData {
pixels: framebuffer.clone(),
})
.is_err()
{
info!("render thread closed, stopping network loop");
break;
}
// Acknowledge the frame.
if let Err(e) = conn.send_frame_ack(update.sequence).await {
warn!(error = %e, "failed to send frame ack");
}
}
Ok(Err(e)) => {
error!(error = %e, "display stream error");
break;
}
Err(_) => {
// Timeout -- no frame available, loop back to drain input.
}
}
}
});
})
.expect("failed to spawn network thread");
// Wait for the network thread to report server dimensions.
let (width, height) = dim_rx
.recv()
.expect("network thread terminated unexpectedly");
if width == 0 || height == 0 {
error!("failed to get server dimensions, exiting");
std::process::exit(1);
}
info!(width, height, "starting display");
// Run the winit event loop on the main thread.
let event_loop = EventLoop::new().expect("failed to create event loop");
let mut app = App::new(width, height, frame_rx, input_tx);
event_loop.run_app(&mut app).expect("event loop error");
}

101
crates/wrsrvd/src/state.rs
View file

@ -22,6 +22,7 @@ use smithay::{
},
};
use tracing::info;
use wayray_protocol::messages::InputMessage;
/// Central compositor state holding all Smithay subsystem states.
///
@ -190,4 +191,104 @@ impl WayRay {
_ => {} // Ignore other events
}
}
/// Inject an input event received from a network client into the
/// compositor's seat, following the same patterns as `process_input_event`.
pub fn inject_network_input(&mut self, msg: InputMessage) {
match msg {
InputMessage::Keyboard(ev) => {
let serial = SERIAL_COUNTER.next_serial();
let keyboard = self.seat.get_keyboard().unwrap();
let state = match ev.state {
wayray_protocol::messages::KeyState::Pressed => {
smithay::backend::input::KeyState::Pressed
}
wayray_protocol::messages::KeyState::Released => {
smithay::backend::input::KeyState::Released
}
};
keyboard.input::<(), _>(
self,
ev.keycode.into(),
state,
serial,
ev.time,
|_, _, _| FilterResult::Forward,
);
}
InputMessage::PointerMotion(ev) => {
let serial = SERIAL_COUNTER.next_serial();
let pointer = self.seat.get_pointer().unwrap();
let pos = (ev.x, ev.y).into();
let under = self.space.element_under(pos).and_then(|(window, loc)| {
window
.surface_under(pos - loc.to_f64(), WindowSurfaceType::ALL)
.map(|(surface, surf_loc)| (surface, (surf_loc + loc).to_f64()))
});
pointer.motion(
self,
under,
&MotionEvent {
location: pos,
serial,
time: ev.time,
},
);
pointer.frame(self);
}
InputMessage::PointerButton(ev) => {
let serial = SERIAL_COUNTER.next_serial();
let pointer = self.seat.get_pointer().unwrap();
let state = match ev.state {
wayray_protocol::messages::ButtonState::Pressed => ButtonState::Pressed,
wayray_protocol::messages::ButtonState::Released => ButtonState::Released,
};
// Click-to-focus on button press.
if state == ButtonState::Pressed {
let pos = pointer.current_location();
if let Some((window, _loc)) = self.space.element_under(pos) {
let window = window.clone();
self.space.raise_element(&window, true);
let keyboard = self.seat.get_keyboard().unwrap();
let wl_surface = window.toplevel().map(|t| t.wl_surface().clone());
keyboard.set_focus(self, wl_surface, serial);
}
}
pointer.button(
self,
&ButtonEvent {
serial,
time: ev.time,
button: ev.button,
state,
},
);
pointer.frame(self);
}
InputMessage::PointerAxis(ev) => {
let pointer = self.seat.get_pointer().unwrap();
let axis = match ev.axis {
wayray_protocol::messages::Axis::Horizontal => Axis::Horizontal,
wayray_protocol::messages::Axis::Vertical => Axis::Vertical,
};
let mut frame = AxisFrame::new(ev.time).source(AxisSource::Wheel);
if ev.value != 0.0 {
frame = frame.value(axis, ev.value);
}
pointer.axis(self, frame);
pointer.frame(self);
}
}
}
}