player/video: treat av sync of large speed changes specially

This is basically just mostly ad hoc from looking at numbers. A large
speed change is defined as a greater than 50% difference between the
previous time frame value and the newly calculated one right after a
speed change (arbitrary). When this is satisfied, there are two distinct
possibilities: the time frame is either negative or positive.

The negative case is actually surprisingly easy to solve. Negative time
frame values are unacceptable since mpv is guaranteed to seek forward
since the audio hasn't caught up yet. So just simply add a tiny negative
value to mpctx->delay (to avoid AV from running away forever) and wait
until the buffer goes positive again before returning back to normal.
This prevents the frames from skipping forwards to weird places for at
least normalish cases.

The positive case is the tricky one. It has a bad tendency to lead to
non-monotonic frame order (i.e. it can skip ahead, then go backwards,
then back forwards again, etc.). This is because the initial frame after
the speed change lingers on the screen for far too long which
essentially causes havoc on the calculations and subsequent passes
through the renderloop overcorrect in both directions until it settles
on the "correct" frame and then proceeds normally.

"Fix" this by basically doing some hacks. Since the source of the
problem is mpctx->time_frame being too big, let's just arbitrarily
reduce the value for a arbitrary amount of frames. Essentially what this
does is smoothen out the change for a short period of time before we
trust that the values are sane enough to allow the normal rendering to
proceed. Up to 8x speed, this seems to work OK for me and the frames
increas monotonically. This is probably about where the limit with this
method is although going any higher will guarentee a/v desync anyways
(you don't actually use speeds this stupid do you).

The final thing here to consider is the display sync code path. It has
similar problems, and the cause in this case is the calculated a_pos
having a dramatic offset from the video pts which causes skipping frames
when changing speed (mostly when you decreaase the speed though). In
this case, what we do is simply hold the a_pos to the video pts until
the a_pos catches back up to a reasonable difference. After that, allow
the normal syncing to happen again.

player/command.c

@@ -7246,6 +7246,7 @@ void mp_option_change_callback(void *ctx, struct m_config_option *co, int flags,
 
  if (opt_ptr == &opts->playback_speed) {
  update_playback_speed(mpctx);
+ mpctx->speed_changed = true;
  mp_wakeup_core(mpctx);
  }
 

player/core.h

@@ -353,6 +353,9 @@ typedef struct MPContext {
  double delay;
  // AV sync: time in seconds until next frame should be shown
  double time_frame;
+ // AV sync: buffer used for time_frame on sudden, large timing changes (i.e.
+ // speed changes)
+ double time_frame_buffer;
  // How much video timing has been changed to make it match the audio
  // timeline. Used for status line information only.
  double total_avsync_change;
@@ -408,6 +411,9 @@ typedef struct MPContext {
  /* Heuristic for potentially redrawing subs. */
  bool redraw_subs;
 
+ bool speed_changed;
+ bool speed_changed_display_sync;
+
  bool paused; // internal pause state
  bool playback_active; // not paused, restarting, loading, unloading
  bool in_playloop;

player/video.c

@@ -617,7 +617,37 @@ static void update_avsync_before_frame(struct MPContext *mpctx)
  buffered_audio = predicted + difference / opts->autosync;
  }
 
- mpctx->time_frame = buffered_audio - mpctx->delay / mpctx->video_speed;
+ double buffer = buffered_audio - mpctx->delay / mpctx->video_speed;
+
+ /* Large speed changes can have a massive, sharp difference which cause
+ * playback position changes if not specially handled. */
+ if (mpctx->speed_changed) {
+ double percent_diff = (buffer - mpctx->time_frame) / ((buffer + mpctx->time_frame) / 2);
+ // arbitrary
+ if (percent_diff > 0.5)
+ mpctx->time_frame_buffer = buffer;
+
+ mpctx->speed_changed = false;
+ }
+
+ if (mpctx->time_frame_buffer > 0) {
+ // Try to avoid by non-monotonic frame orders by limiting the buffer
+ // by an arbitrary factor for an equally arbitrary amount of frames.
+ // Probably won't work for insanely high speeds that cause an a/v
+ // desync but oh well.
+ mpctx->time_frame_buffer -= mpctx->time_frame_buffer * 0.01;
+ mpctx->time_frame = buffer * 0.5;
+ if (mpctx->time_frame_buffer <= 0.05)
+ mpctx->time_frame_buffer = 0;
+ } else if (mpctx->time_frame_buffer < 0) {
+ // Add a small, arbitrary negative offset to avoid seeking forwards.
+ // Waith until buffer becomes positive again before breaking.
+ mpctx->time_frame = -0.01; //
+ if (buffer > 0)
+ mpctx->time_frame_buffer = 0;
+ } else {
+ mpctx->time_frame = buffer;
+ }
  } else {
  /* If we're more than 200 ms behind the right playback
  * position, don't try to speed up display of following
@@ -648,6 +678,20 @@ static void update_av_diff(struct MPContext *mpctx, double offset)
  double a_pos = written_audio_pts(mpctx);
  if (a_pos != MP_NOPTS_VALUE && mpctx->video_pts != MP_NOPTS_VALUE) {
  a_pos -= mpctx->audio_speed * ao_get_delay(mpctx->ao);
+
+ // On large speed changes, a_pos will drift a ton away from the video
+ // pts which causes the playback position to change. Workaround this by
+ // clamping it to the video pts until a_pos gets back to a reasonable
+ // value.
+ if (mpctx->speed_changed_display_sync && mpctx->num_past_frames == MAX_NUM_VO_PTS) {
+ double percent_diff = (a_pos - mpctx->video_pts) / ((a_pos + mpctx->video_pts) / 2);
+ if (percent_diff > 0.1) {
+ a_pos = mpctx->video_pts;
+ } else {
+ mpctx->speed_changed_display_sync = false;
+ }
+ }
+
  mpctx->last_av_difference = a_pos - mpctx->video_pts
  + opts->audio_delay + offset;
  }
@@ -829,6 +873,13 @@ static void handle_display_sync_frame(struct MPContext *mpctx,
  mode == VS_DISP_TEMPO;
  drop &= frame->can_drop;
 
+ // Return here so the old frame_duration doesn't get used.
+ if (mpctx->speed_changed && !mpctx->speed_changed_display_sync) {
+ mpctx->speed_changed = false;
+ mpctx->speed_changed_display_sync = true;
+ return;
+ }
+
  if (resample && using_spdif_passthrough(mpctx))
  return;