Update frame selection with respect to spectator view ping time. (#159)

SpectatorView/Compositor/CompositorDLL/VideoEncoder.cpp

@@ -37,6 +37,8 @@ bool VideoEncoder::Initialize(ID3D11Device* device)
  HRESULT hr = E_PENDING;
  hr = MFStartup(MF_VERSION);
 
+ QueryPerformanceFrequency(&freq);
+
 #if HARDWARE_ENCODE_VIDEO
  MFCreateDXGIDeviceManager(&resetToken, &deviceManager);
 
@@ -179,9 +181,6 @@ void VideoEncoder::WriteAudio(byte* buffer, LONGLONG timestamp)
  return;
  }
 
- LARGE_INTEGER freq;
- QueryPerformanceFrequency(&freq);
-
  LONGLONG sampleTimeNow = timestamp;
  if (sampleTimeNow < 0) { sampleTimeNow *= -1; }
  LONGLONG sampleTimeStart = startTime;
@@ -267,9 +266,6 @@ void VideoEncoder::WriteVideo(byte* buffer, LONGLONG timestamp, LONGLONG duratio
  return;
  }
 
- LARGE_INTEGER freq;
- QueryPerformanceFrequency(&freq);
-
  LONGLONG sampleTimeNow = timestamp;
  if (sampleTimeNow < 0) { sampleTimeNow *= -1; }
  LONGLONG sampleTimeStart = startTime;

SpectatorView/Compositor/CompositorDLL/VideoEncoder.h

@@ -48,6 +48,8 @@ class VideoEncoder
  void WriteVideo(byte* buffer, LONGLONG timestamp, LONGLONG duration);
  void WriteAudio(byte* buffer, LONGLONG timestamp);
 
+ LARGE_INTEGER freq;
+
  class VideoInput
  {
  public:

SpectatorView/Compositor/UnityCompositorInterface/UnityCompositorInterface.cpp

@@ -189,7 +189,7 @@ UNITYDLL void GetEarliestHologramPose(
 // Set hologram poses when we get them on the network.
 UNITYDLL void SetHologramPose(
  float rotX, float rotY, float rotZ, float rotW,
- float posX, float posY, float posZ)
+ float posX, float posY, float posZ, float msOffset)
 {
  if (ci == nullptr)
  {
@@ -201,7 +201,10 @@ UNITYDLL void SetHologramPose(
 
  LONGLONG timestamp = time.QuadPart;
 
- auto hologramFrame = ci->GetNextHologramFrame(timestamp);
+ // Convert offset to microseconds.
+ LONGLONG offset = (LONGLONG)(msOffset * 1000.0f);
+
+ auto hologramFrame = ci->GetNextHologramFrame(timestamp - offset);
  if (hologramFrame != nullptr)
  {
  hologramFrame->rotX = rotX;
@@ -240,11 +243,9 @@ UNITYDLL void UpdateSpectatorView()
  if (cachedTime != colorTime)
  {
  LONGLONG frameDuration = ci->GetColorDuration();
- float buffer = 0.1f * frameDuration;
-
  // Find a pose on the leading end of this color frame.
  const auto frame = ci->FindClosestHologramFrame(
- colorTime - frameDuration + buffer, (LONGLONG)(_frameOffset * (float)frameDuration));
+ colorTime - frameDuration, (LONGLONG)(_frameOffset * (float)frameDuration));
 
  if (frame != nullptr)
  {

SpectatorView/Samples/SharedHolograms/Assets/Addons/HolographicCameraRig/SV_UNET/Scripts/PlayerController.cs

@@ -26,7 +26,7 @@ private static extern bool GetHeadTransform(System.IntPtr unityCoordinateSystem,
  [DllImport("UnityCompositorInterface")]
  private static extern bool SetHologramPose(
  float rotX, float rotY, float rotZ, float rotW,
- float posX, float posY, float posZ);
+ float posX, float posY, float posZ, float msOffset);
 #endif
  #endregion
 
@@ -40,7 +40,6 @@ public override float GetNetworkSendInterval()
  return 0.033f;
  }
 
-
  private static PlayerController _Instance = null;
  public static PlayerController Instance
  {
@@ -73,6 +72,9 @@ public bool CanEstablishAnchor()
  [SyncVar]
  private Vector3 localPosition;
 
+ [SyncVar]
+ int spectatorViewPing = 0;
+
  /// <summary>
  /// The rotation relative to the shared world anchor.
  /// </summary>
@@ -109,10 +111,15 @@ public void SetColorDuration(long value)
  /// <param name="postion">the localPosition to set</param>
  /// <param name="rotation">the localRotation to set</param>
  [Command(channel = 1)]
- public void CmdTransform(Vector3 postion, Quaternion rotation)
+ public void CmdTransform(Vector3 postion, Quaternion rotation, int ping)
  {
  localPosition = postion;
  localRotation = rotation;
+
+ if (IsSV())
+ {
+ spectatorViewPing = ping;
+ }
  }
 
  [SyncVar(hook = "AnchorEstablishedChanged")]
@@ -612,7 +619,7 @@ private void Update()
  if (IsSV())
  {
  SetHologramPose(localRotation.x, localRotation.y, localRotation.z, localRotation.w,
- localPosition.x, localPosition.y, localPosition.z);
+ localPosition.x, localPosition.y, localPosition.z, (float)spectatorViewPing / 2.0f);
  }
 #endif
  return;
@@ -657,9 +664,15 @@ private void Update()
  transform.rotation = Camera.main.transform.rotation;
  }
 
- // For UNET we use a command to signal the host to update our local position
- // and rotation
- CmdTransform(transform.localPosition, transform.localRotation);
+ // For UNET we use a command to signal the host to update our local position and rotation
+ if (IsSV())
+ {
+ CmdTransform(transform.localPosition, transform.localRotation, NetworkManager.singleton.client.GetRTT());
+ }
+ else
+ {
+ CmdTransform(transform.localPosition, transform.localRotation, 0);
+ }
  }
 
  /// <summary>