feat: virtual keyboard

.github/workflows/build.yml

@@ -106,7 +106,6 @@ jobs:
         key: ${{ runner.os }}-cargo-${{ hashFiles('Cargo.lock') }}
     - name: Download deps
       run: |
-        npm install -g typescript
         brew install nasm
         cargo install cargo-bundle
       shell: bash

.gitignore

@@ -7,6 +7,11 @@
 /deps/libva
 c_helper/target
 c_helper/Cargo.lock
+.DS_Store
+node_modules
+/www/static/lib.js
+/www/static/style.css
+/www/static/*.map
 *.js
 *.js.map
 *.tar.gz

CONTRIBUTORS

@@ -7,6 +7,7 @@ List of Contributors:
 **************************
 Robert Schroll
 Daniel Rutz
+lyonbot
 Philipp Urlbauer
 OmegaRogue
 **************************

Cargo.toml

@@ -7,7 +7,7 @@ edition = "2021"
 description = "Use your iPad or Android tablet as graphic tablet."
 
 [dependencies]
-autopilot = { git = "https://github.com/H-M-H/autopilot-rs.git", rev = "63eed09c715bfb665bb23172a3930a528e11691c" }
+autopilot = { git = "https://github.com/lyonbot/autopilot-rs.git", rev = "a92b41edb4818d0d4b4289d49e65420dbe9ab081", version = "0.4.0" }
 bitflags = { version = "^2.6", features = ["serde"] }
 bytes = "1.7.1"
 clap = { version = "4.5.18", features = ["derive"] }

Readme.md

@@ -260,7 +260,7 @@ Weylus can make use of Nvidias NVENC as well as Microsoft's MediaFoundation for
 video encoding. Due to widely varying quality it is disabled by default.
 
 ## Building
-To build Weylus you need to install Rust, Typescript, make, git, a C compiler, nasm and bash. `cargo
+To build Weylus you need to install Rust, NodeJS, make, git, a C compiler, nasm and bash. `cargo
 build` builds the project. By default Weylus is build in debug mode, if you want a release build run
 `cargo build --release`. On Linux some additional dependencies are required to build Weylus. On
 Debian or Ubuntu they can be installed via:
@@ -278,10 +278,6 @@ libXfixes-devel libXtst-devel libXrandr-devel libXcomposite-devel libXi-devel li
 pkg-config libdrm-devel pango-devel gstreamer1-devel \
 gstreamer1-plugins-base-devel dbus-devel nasm npm
 ```
-After npm is installed, typescript must be installed by:
-```sh
-sudo npm install typescript -g
-```
 
 Note that building for the first time may take a while as by default ffmpeg needs to be built. On
 Windows only msvc is supported as C compiler; it is, however, possible to cross compile on Linux for

build.rs

@@ -26,6 +26,64 @@ fn build_ffmpeg(dist_dir: &Path) {
     }
 }
 
+fn build_www() {
+    let www_dir = Path::new("www");
+
+    #[cfg(not(target_os = "windows"))]
+    let shell = "bash";
+
+    #[cfg(not(target_os = "windows"))]
+    let shell_flag = "-c";
+
+    #[cfg(target_os = "windows")]
+    let shell = "cmd";
+
+    #[cfg(target_os = "windows")]
+    let shell_flag = "/c";
+
+
+    // try `pnpm` first, then `npm`
+    if !www_dir.join("node_modules").exists() {
+        let pnpm_install_success = match Command::new(shell)
+            .args([shell_flag, "pnpm install"])
+            .current_dir(www_dir)
+            .status()
+        {
+            Ok(e) => e.success(),
+            Err(_) => false,
+        };
+
+        if !pnpm_install_success {
+            let npm_install_result = Command::new(shell)
+                .args([shell_flag, "npm install"])
+                .current_dir(www_dir)
+                .status()
+                .expect("Failed to run npm or pnpm!");
+
+            if !npm_install_result.success() {
+                panic!(
+                    "Failed to install npm dependencies! npm exited with code {}",
+                    npm_install_result.code().unwrap_or(-1)
+                );
+            }
+        }
+    }
+
+    let build_result = Command::new(shell)
+        .args([shell_flag, "npm run build"])
+        .current_dir(www_dir)
+        .status()
+        .expect("Failed to build www!");
+
+    if !build_result.success() {
+        panic!(
+            "Failed to build www! npm exited with code {}",
+            build_result.code().unwrap_or(-1)
+        );
+    }
+}
+
+
 fn main() {
     let target_os = env::var("CARGO_CFG_TARGET_OS").unwrap();
 
@@ -38,39 +96,8 @@ fn main() {
         build_ffmpeg(&dist_dir);
     }
 
-    println!("cargo:rerun-if-changed=ts/lib.ts");
-
-    #[cfg(not(target_os = "windows"))]
-    let mut tsc_command = Command::new("tsc");
-
-    #[cfg(target_os = "windows")]
-    let mut tsc_command = Command::new("bash");
-    #[cfg(target_os = "windows")]
-    tsc_command.args(&["-c", "tsc"]);
-
-    let js_needs_update = || -> Result<bool, Box<dyn std::error::Error>> {
-        Ok(Path::new("ts/lib.ts").metadata()?.modified()?
-            > Path::new("www/static/lib.js").metadata()?.modified()?)
-    }()
-    .unwrap_or(true);
-
-    if js_needs_update {
-        match tsc_command.status() {
-            Err(err) => {
-                println!("cargo:warning=Failed to call tsc: {}", err);
-                std::process::exit(1);
-            }
-            Ok(status) => {
-                if !status.success() {
-                    match status.code() {
-                        Some(code) => println!("cargo:warning=tsc failed with exitcode: {}", code),
-                        None => println!("cargo:warning=tsc terminated by signal."),
-                    };
-                    std::process::exit(2);
-                }
-            }
-        }
-    }
+    println!("cargo:rerun-if-changed=www/src/");
+    build_www();
 
     println!("cargo:rerun-if-changed=lib/encode_video.c");
     let mut cc_video = cc::Build::new();

docker/Dockerfile

@@ -16,8 +16,7 @@ RUN curl -LO "https://www.nasm.us/pub/nasm/releasebuilds/2.16.03/nasm-2.16.03.ta
     tar xf "nasm-2.16.03.tar.xz" && cd "nasm-2.16.03" && \
     ./configure --prefix=/usr && make -j$(nproc) && make install && cd .. && rm -rf "nasm-2.16.03*"
 RUN curl -sL https://deb.nodesource.com/setup_20.x | bash - && \
-    apt-get install -y nodejs && \
-    npm install -g typescript
+    apt-get install -y nodejs
 RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | \
     sh -s -- -y --default-toolchain stable-x86_64-unknown-linux-gnu
 RUN cargo install cargo-deb

docker/Dockerfile_alpine

@@ -6,8 +6,6 @@ RUN apk add --no-cache libx11-dev libxext-dev libxft-dev libxinerama-dev libxcur
     autoconf libtool pkgconfig libdrm-dev pango-dev gst-plugins-base-dev gstreamer-dev dbus-libs \
     dbus-dev cmake build-base nasm npm ffmpeg-dev libva-dev curl git bash automake tar
 
-RUN npm install --global typescript
-
 RUN ln -s /usr/bin/x86_64-alpine-linux-musl-gcc /usr/bin/musl-gcc
 RUN ln -s /usr/bin/x86_64-alpine-linux-musl-g++ /usr/bin/musl-g++
 

src/config.rs

@@ -127,6 +127,8 @@ pub struct Config {
     #[serde(skip)]
     pub print_lib_js: bool,
 
+    pub virtual_keys_profiles: Option<String>,
+
     #[arg(
         long,
         help = "Use custom template of index.html to be served by Weylus."

src/input/autopilot_device.rs

@@ -3,20 +3,30 @@ use autopilot::mouse;
 use autopilot::mouse::ScrollDirection;
 use autopilot::screen::size as screen_size;
 
-use tracing::warn;
+use tracing::{debug, warn};
 
 use crate::input::device::{InputDevice, InputDeviceType};
-use crate::protocol::{Button, KeyboardEvent, KeyboardEventType, PointerEvent, WheelEvent};
+use crate::protocol::{
+    Button, KeyboardEvent, KeyboardEventType, PointerEvent, PointerEventType, PointerType,
+    WheelEvent,
+};
 
 use crate::capturable::{Capturable, Geometry};
 
+#[cfg(target_os = "macos")]
+use super::macos_tablet::{MacosPenEventType, macos_send_tablet_event};
+
 pub struct AutoPilotDevice {
+    tablet_down: bool,
     capturable: Box<dyn Capturable>,
 }
 
 impl AutoPilotDevice {
     pub fn new(capturable: Box<dyn Capturable>) -> Self {
-        Self { capturable }
+        Self {
+            tablet_down: false,
+            capturable,
+        }
     }
 }
 
@@ -30,7 +40,7 @@ impl InputDevice for AutoPilotDevice {
     }
 
     fn send_pointer_event(&mut self, event: &PointerEvent) {
-        if !event.is_primary {
+        if !event.is_primary && event.pointer_type != PointerType::Pen {
             return;
         }
         if let Err(err) = self.capturable.before_input() {
@@ -54,12 +64,62 @@ impl InputDevice for AutoPilotDevice {
                 return;
             }
         };
-        if let Err(err) = mouse::move_to(autopilot::geometry::Point::new(
+
+        let point = autopilot::geometry::Point::new(
             (event.x * width_rel + x_rel) * width,
             (event.y * height_rel + y_rel) * height,
-        )) {
+        );
+
+        // MacOS only: send tablet (stylus) events
+        #[cfg(target_os = "macos")]
+        if event.pointer_type == PointerType::Pen {
+            let pe_type = match event.event_type {
+                PointerEventType::DOWN => MacosPenEventType::Down,
+                PointerEventType::UP => MacosPenEventType::Up,
+                PointerEventType::CANCEL => MacosPenEventType::Up,
+                PointerEventType::ENTER => MacosPenEventType::Enter,
+                PointerEventType::LEAVE => MacosPenEventType::Leave,
+                _ => MacosPenEventType::Move,
+            };
+
+            match event.event_type {
+                PointerEventType::DOWN => {
+                    self.tablet_down = true;
+                }
+                PointerEventType::CANCEL | PointerEventType::UP | PointerEventType::LEAVE => {
+                    self.tablet_down = false;
+                }
+                _ => (),
+            }
+
+            match event.event_type {
+                PointerEventType::ENTER => {
+                    debug!("Entering tablet");
+                }
+                PointerEventType::LEAVE => {
+                    debug!("Leaving tablet");
+                }
+                _ => (),
+            }
+
+            let buttons = if self.tablet_down { 1 } else { 0 };
+            if let Err(err) = macos_send_tablet_event(
+                point,
+                pe_type,
+                event.button.bits().into(),
+                buttons,
+                event.pressure,
+            ) {
+                warn!("Could not send pressure: {}", err);
+            }
+
+            return;
+        }
+
+        if let Err(err) = mouse::move_to(point) {
             warn!("Could not move mouse: {}", err);
         }
+
         match event.button {
             Button::PRIMARY => {
                 mouse::toggle(mouse::Button::Left, event.buttons.contains(event.button))

src/input/autopilot_device_win.rs

@@ -57,13 +57,23 @@ impl InputDevice for WindowsInput {
             (event.y * height as f64) as i32 + offset_y,
         );
         let mut pointer_flags = match event.event_type {
+            PointerEventType::ENTER => {
+                POINTER_FLAG_INRANGE | POINTER_FLAG_NEW
+            }
             PointerEventType::DOWN => {
                 POINTER_FLAG_INRANGE | POINTER_FLAG_INCONTACT | POINTER_FLAG_DOWN
             }
-            PointerEventType::MOVE => POINTER_FLAG_INRANGE | POINTER_FLAG_UPDATE,
-            PointerEventType::UP => POINTER_FLAG_UP,
+            PointerEventType::MOVE => {
+                POINTER_FLAG_INRANGE | POINTER_FLAG_UPDATE // POINTER_FLAG_INCONTACT see below "buttons" part
+            }
+            PointerEventType::UP => {
+                POINTER_FLAG_INRANGE | POINTER_FLAG_UP
+            }
             PointerEventType::CANCEL => {
-                POINTER_FLAG_INRANGE | POINTER_FLAG_UPDATE | POINTER_FLAG_CANCELED
+                POINTER_FLAG_CANCELED | POINTER_FLAG_UP
+            }
+            PointerEventType::LEAVE => {
+                POINTER_FLAG_NONE // anything but POINTER_FLAG_INRANGE
             }
         };
         let button_change_type = match event.buttons {
@@ -157,6 +167,10 @@ impl InputDevice for WindowsInput {
                         PointerEventType::UP | PointerEventType::CANCEL => {
                             self.multitouch_map.remove(&event.pointer_id);
                         }
+
+                        PointerEventType::ENTER | PointerEventType::LEAVE => {
+                            // nothing to do with touch
+                        }
                     }
                 }
             }
@@ -198,6 +212,9 @@ impl InputDevice for WindowsInput {
                     },
                     PointerEventType::CANCEL => {
                         dw_flags |= MOUSEEVENTF_LEFTUP;
+                    },
+                    PointerEventType::ENTER | PointerEventType::LEAVE => {
+                        // nothing to do with mouse
                     }
                 }
                 unsafe { mouse_event(dw_flags, 0 as u32, 0 as u32, 0, 0) };