Add --hid-capture & --hid-replay options

app/meson.build

@@ -88,8 +88,11 @@ usb_support = get_option('usb')
 if usb_support
  src += [
  'src/usb/aoa_hid.c',
+ 'src/usb/hid_event.c',
+ 'src/usb/hid_event_serializer.c',
  'src/usb/hid_keyboard.c',
  'src/usb/hid_mouse.c',
+ 'src/usb/hid_replay.c',
  'src/usb/scrcpy_otg.c',
  'src/usb/screen_otg.c',
  'src/usb/usb.c',
@@ -257,6 +260,16 @@ if get_option('buildtype') == 'debug'
  'tests/test_vector.c',
  ]],
  ]
+ if usb_support
+ tests += [
+ ['test_hid_event_serializer', [
+ 'tests/test_hid_event_serializer.c',
+ 'src/usb/hid_event.c',
+ 'src/usb/hid_event_serializer.c',
+ 'src/util/memory.c',
+ ]],
+ ]
+ endif
 
  foreach t : tests
  sources = t[1] + ['src/compat.c']

app/scrcpy.1

@@ -171,6 +171,23 @@ By default, right-click triggers BACK (or POWER on) and middle-click triggers HO
 .B \-h, \-\-help
 Print this help.
 
+.TP
+.BI "\-\-hid-record " output.log
+Record HID input to a file.
+
+Only HID input is recorded, which requires the \fB\-\-hid\-keyboard\fR, \fB\-\-hid\-mouse\fR and/or \fB\-\-otg\fR options.
+When \fB\-\-hid\-replay\fR is used simulatenously, any replayed input is also recorded to the target specified by \fB\-\-hid\-replay\fR.
+
+To mirror input to multiple devices, specify a named pipe as the filename (see mkfifo for UNIX) and pass the same filename to \fB\-\-hid\-replay\fR of a second (parallel) scrcpy instance.
+
+.TP
+.BI "\-\-hid-replay " input.log
+Replay HID input from a file created by \fB\-\-hid-record\fR.
+
+Events are only replayed when HID is used to simulate input, which requires the \fB\-\-hid\-keyboard\fR, \fB\-\-hid\-mouse\fR and/or \fB\-\-otg\fR options.
+
+See \fB\-\-hid\-record\fR for more information.
+
 .TP
 .B \-\-kill\-adb\-on\-close
 Kill adb when scrcpy terminates.

app/src/cli.c

@@ -58,6 +58,8 @@ enum {
  OPT_RAW_KEY_EVENTS,
  OPT_NO_DOWNSIZE_ON_ERROR,
  OPT_OTG,
+ OPT_HID_RECORD,
+ OPT_HID_REPLAY,
  OPT_NO_CLEANUP,
  OPT_PRINT_FPS,
  OPT_NO_POWER_ON,
@@ -358,6 +360,29 @@ static const struct sc_option options[] = {
  .longopt = "help",
  .text = "Print this help.",
  },
+ {
+ .longopt_id = OPT_HID_RECORD,
+ .longopt = "hid-record",
+ .argdesc = "output.log",
+ .text = "Record HID input to a file.\n"
+ "Only HID input is recorded, which requires the --hid-keyboard,"
+ "--hid-mouse and/or --otg options.\n"
+ "When --hid-replay is used simulatenously, any replayed input "
+ "is also recorded to the target specified by --hid-replay.\n"
+ "To mirror input to multiple devices, specify a named pipe as "
+ "the filename (see mkfifo for UNIX) and pass the same filename "
+ "to --hid-replay of a second (parallel) scrcpy instance.",
+ },
+ {
+ .longopt_id = OPT_HID_REPLAY,
+ .longopt = "hid-replay",
+ .argdesc = "input.log",
+ .text = "Replay HID input from a file created by --hid-record.\n"
+ "Events are only replayed when HID is used to simulate input,"
+ "which requires the --hid-keyboard, --hid-mouse and/or --otg "
+ "options.\n"
+ "See --hid-record for more information.",
+ },
  {
  .longopt_id = OPT_KILL_ADB_ON_CLOSE,
  .longopt = "kill-adb-on-close",
@@ -2267,6 +2292,22 @@ parse_args_with_getopt(struct scrcpy_cli_args *args, int argc, char *argv[],
 #else
  LOGE("OTG mode (--otg) is disabled.");
  return false;
+#endif
+ case OPT_HID_RECORD:
+#ifdef HAVE_USB
+ opts->hid_record_filename = optarg;
+ break;
+#else
+ LOGE("HID recording (--hid-record) is disabled.");
+ return false;
+#endif
+ case OPT_HID_REPLAY:
+#ifdef HAVE_USB
+ opts->hid_replay_filename = optarg;
+ break;
+#else
+ LOGE("HID replay (--hid-replay) is disabled.");
+ return false;
 #endif
  case OPT_V4L2_SINK:
 #ifdef HAVE_V4L2
@@ -2628,6 +2669,16 @@ parse_args_with_getopt(struct scrcpy_cli_args *args, int argc, char *argv[],
  return false;
  }
 # endif
+#ifdef HAVE_USB
+ if (opts->hid_replay_filename || opts->hid_record_filename) {
+ if (!otg && opts->keyboard_input_mode != SC_KEYBOARD_INPUT_MODE_HID
+ && opts->mouse_input_mode != SC_MOUSE_INPUT_MODE_HID) {
+ LOGE("--hid-record and --hid-replay only works if --hid-keyboard, "
+ "--hid-mouse and/or --otg are set.");
+ return false;
+ }
+ }
+#endif
 
  if (otg) {
  // OTG mode is compatible with only very few options.

app/src/options.c

@@ -56,6 +56,8 @@ const struct scrcpy_options scrcpy_options_default = {
 #endif
 #ifdef HAVE_USB
  .otg = false,
+ .hid_record_filename = NULL,
+ .hid_replay_filename = NULL,
 #endif
  .show_touches = false,
  .fullscreen = false,

app/src/options.h

@@ -236,6 +236,8 @@ struct scrcpy_options {
 #endif
 #ifdef HAVE_USB
  bool otg;
+ const char *hid_record_filename;
+ const char *hid_replay_filename;
 #endif
  bool show_touches;
  bool fullscreen;

app/src/scrcpy.c

@@ -29,6 +29,7 @@
 # include "usb/aoa_hid.h"
 # include "usb/hid_keyboard.h"
 # include "usb/hid_mouse.h"
+# include "usb/hid_replay.h"
 # include "usb/usb.h"
 #endif
 #include "util/acksync.h"
@@ -59,6 +60,7 @@ struct scrcpy {
 #ifdef HAVE_USB
  struct sc_usb usb;
  struct sc_aoa aoa;
+ struct sc_hidr hidr;
  // sequence/ack helper to synchronize clipboard and Ctrl+v via HID
  struct sc_acksync acksync;
 #endif
@@ -332,6 +334,11 @@ scrcpy(struct scrcpy_options *options) {
  bool aoa_hid_initialized = false;
  bool hid_keyboard_initialized = false;
  bool hid_mouse_initialized = false;
+ bool hidr_initialized = false;
+ bool hidr_replay_started = false;
+ bool hidr_record_started = false;
+ bool need_hidr = options->hid_record_filename ||
+ options->hid_replay_filename;
 #endif
  bool controller_initialized = false;
  bool controller_started = false;
@@ -609,6 +616,28 @@ scrcpy(struct scrcpy_options *options) {
  }
 
  bool need_aoa = hid_keyboard_initialized || hid_mouse_initialized;
+ if (need_hidr) {
+ if (!need_aoa) {
+ goto end;
+ }
+ ok = sc_hidr_init(&s->hidr, &s->aoa,
+ options->hid_record_filename,
+ options->hid_replay_filename, hid_keyboard_initialized,
+ hid_mouse_initialized);
+ if (!ok) {
+ goto end;
+ }
+ hidr_initialized = true;
+ }
+ if (options->hid_record_filename) {
+ // Set up hidr record BEFORE starting aoa, to ensure that hidr
+ // can listen to events from aoa before the aoa thread starts.
+ ok = sc_hidr_start_record(&s->hidr);
+ if (!ok) {
+ goto end;
+ }
+ hidr_record_started = true;
+ }
 
  if (!need_aoa || !sc_aoa_start(&s->aoa)) {
  sc_acksync_destroy(&s->acksync);
@@ -632,6 +661,9 @@ scrcpy(struct scrcpy_options *options) {
  sc_hid_mouse_destroy(&s->mouse_hid);
  hid_mouse_initialized = false;
  }
+ if (need_hidr) {
+ goto end;
+ }
  }
 
  if (use_hid_keyboard && !hid_keyboard_initialized) {
@@ -681,6 +713,12 @@ scrcpy(struct scrcpy_options *options) {
  // There is a controller if and only if control is enabled
  assert(options->control == !!controller);
 
+#ifdef HAVE_USB
+ if (need_hidr && !aoa_hid_initialized) {
+ goto end;
+ }
+#endif
+
  if (options->video_playback) {
  const char *window_title =
  options->window_title ? options->window_title : info->device_name;
@@ -799,6 +837,17 @@ scrcpy(struct scrcpy_options *options) {
  timeout_started = true;
  }
 
+#ifdef HAVE_USB
+ if (hidr_initialized && options->hid_replay_filename) {
+ assert(aoa_hid_initialized);
+ bool ok = sc_hidr_start_replay(&s->hidr);
+ if (!ok) {
+ goto end;
+ }
+ hidr_replay_started = true;
+ }
+#endif
+
  ret = event_loop(s);
  LOGD("quit...");
 
@@ -814,6 +863,13 @@ scrcpy(struct scrcpy_options *options) {
  // The demuxer is not stopped explicitly, because it will stop by itself on
  // end-of-stream
 #ifdef HAVE_USB
+ if (need_hidr && !aoa_hid_initialized) {
+ LOGE("HID input unavailable, cannot use --hid-record / --hid-replay!");
+ }
+ if (hidr_replay_started) {
+ sc_hidr_stop_replay(&s->hidr);
+ // hidr will not call aoa at this point.
+ }
  if (aoa_hid_initialized) {
  if (hid_keyboard_initialized) {
  sc_hid_keyboard_destroy(&s->keyboard_hid);
@@ -877,6 +933,15 @@ scrcpy(struct scrcpy_options *options) {
  sc_usb_disconnect(&s->usb);
  sc_usb_destroy(&s->usb);
  }
+ if (hidr_record_started) {
+ // Because aoa has been destroyed, aoa won't call hidr at this point.
+ sc_hidr_stop_record(&s->hidr);
+ }
+ if (hidr_initialized) {
+ // aoa has been destroyed, there are no circular references between
+ // hidr and aoa, so we can finally destroy hidr.
+ sc_hidr_destroy(&s->hidr);
+ }
 #endif
 
  // Destroy the screen only after the video demuxer is guaranteed to be

app/src/usb/aoa_hid.c

@@ -4,7 +4,7 @@
 #include <stdio.h>
 
 #include "aoa_hid.h"
-#include "util/log.h"
+#include "hid_replay.h"
 
 // See <https://source.android.com/devices/accessories/aoa2#hid-support>.
 #define ACCESSORY_REGISTER_HID 54
@@ -33,20 +33,6 @@ sc_hid_event_log(const struct sc_hid_event *event) {
  free(buffer);
 }
 
-void
-sc_hid_event_init(struct sc_hid_event *hid_event, uint16_t accessory_id,
- unsigned char *buffer, uint16_t buffer_size) {
- hid_event->accessory_id = accessory_id;
- hid_event->buffer = buffer;
- hid_event->size = buffer_size;
- hid_event->ack_to_wait = SC_SEQUENCE_INVALID;
-}
-
-void
-sc_hid_event_destroy(struct sc_hid_event *hid_event) {
- free(hid_event->buffer);
-}
-
 bool
 sc_aoa_init(struct sc_aoa *aoa, struct sc_usb *usb,
  struct sc_acksync *acksync) {
@@ -70,6 +56,9 @@ sc_aoa_init(struct sc_aoa *aoa, struct sc_usb *usb,
  aoa->stopped = false;
  aoa->acksync = acksync;
  aoa->usb = usb;
+ // If needed, will be initialized by sc_hidr_start_record,
+ // before a new thread is created by sc_aoa_start.
+ aoa->hidr_to_notify = NULL;
 
  return true;
 }
@@ -219,6 +208,9 @@ sc_aoa_push_hid_event(struct sc_aoa *aoa, const struct sc_hid_event *event) {
  if (sc_get_log_level() <= SC_LOG_LEVEL_VERBOSE) {
  sc_hid_event_log(event);
  }
+ if (aoa->hidr_to_notify) {
+ sc_hidr_observe_event_for_record(aoa->hidr_to_notify, event);
+ }
 
  sc_mutex_lock(&aoa->mutex);
  bool full = sc_vecdeque_is_full(&aoa->queue);
@@ -232,6 +224,13 @@ sc_aoa_push_hid_event(struct sc_aoa *aoa, const struct sc_hid_event *event) {
  // Otherwise (if the queue is full), the event is discarded
 
  sc_mutex_unlock(&aoa->mutex);
+ if (full) {
+ // In the not-full case, a copy is made of the struct, which includes the
+ // heap-allocated event->buffer member. The receiver of that data will
+ // take care of cleaning it up.
+ // We need to clean up similarly when the data is dropped.
+ sc_hid_event_destroy((struct sc_hid_event *)event);
+ }
 
  return !full;
 }

app/src/usb/aoa_hid.h

@@ -6,29 +6,17 @@
 
 #include <libusb-1.0/libusb.h>
 
+#include "hid_event.h"
 #include "usb.h"
 #include "util/acksync.h"
 #include "util/thread.h"
-#include "util/tick.h"
 #include "util/vecdeque.h"
 
-struct sc_hid_event {
- uint16_t accessory_id;
- unsigned char *buffer;
- uint16_t size;
- uint64_t ack_to_wait;
-};
-
-// Takes ownership of buffer
-void
-sc_hid_event_init(struct sc_hid_event *hid_event, uint16_t accessory_id,
- unsigned char *buffer, uint16_t buffer_size);
-
-void
-sc_hid_event_destroy(struct sc_hid_event *hid_event);
-
 struct sc_hid_event_queue SC_VECDEQUE(struct sc_hid_event);
 
+// Forward declare sc_hidr to avoid circular dependency on hid_replay.h.
+struct sc_hidr;
+
 struct sc_aoa {
  struct sc_usb *usb;
  sc_thread thread;
@@ -38,6 +26,7 @@ struct sc_aoa {
  struct sc_hid_event_queue queue;
 
  struct sc_acksync *acksync;
+ struct sc_hidr *hidr_to_notify;
 };
 
 bool