Update frame selection with respect to spectator view ping time. (microsoft#159)

Author: fieldsJacksonG

SpectatorView/Compositor/CompositorDLL/VideoEncoder.cpp

@@ -37,6 +37,8 @@ bool VideoEncoder::Initialize(ID3D11Device* device)
     HRESULT hr = E_PENDING;
     hr = MFStartup(MF_VERSION);
 
+    QueryPerformanceFrequency(&freq);
+
 #if HARDWARE_ENCODE_VIDEO
     MFCreateDXGIDeviceManager(&resetToken, &deviceManager);
 
@@ -179,9 +181,6 @@ void VideoEncoder::WriteAudio(byte* buffer, LONGLONG timestamp)
         return;
     }
 
-    LARGE_INTEGER freq;
-    QueryPerformanceFrequency(&freq);
-
     LONGLONG sampleTimeNow = timestamp;
     if (sampleTimeNow < 0) { sampleTimeNow *= -1; }
     LONGLONG sampleTimeStart = startTime;
@@ -267,9 +266,6 @@ void VideoEncoder::WriteVideo(byte* buffer, LONGLONG timestamp, LONGLONG duratio
         return;
     }
 
-    LARGE_INTEGER freq;
-    QueryPerformanceFrequency(&freq);
-
     LONGLONG sampleTimeNow = timestamp;
     if (sampleTimeNow < 0) { sampleTimeNow *= -1; }
     LONGLONG sampleTimeStart = startTime;

SpectatorView/Compositor/CompositorDLL/VideoEncoder.h

@@ -48,6 +48,8 @@ class VideoEncoder
     void WriteVideo(byte* buffer, LONGLONG timestamp, LONGLONG duration);
     void WriteAudio(byte* buffer, LONGLONG timestamp);
 
+    LARGE_INTEGER freq;
+
     class VideoInput
     {
     public:

SpectatorView/Compositor/UnityCompositorInterface/UnityCompositorInterface.cpp

@@ -189,7 +189,7 @@ UNITYDLL void GetEarliestHologramPose(
 // Set hologram poses when we get them on the network.
 UNITYDLL void SetHologramPose(
     float rotX, float rotY, float rotZ, float rotW,
-    float posX, float posY, float posZ)
+    float posX, float posY, float posZ, float msOffset)
 {
     if (ci == nullptr)
     {
@@ -201,7 +201,10 @@ UNITYDLL void SetHologramPose(
 
     LONGLONG timestamp = time.QuadPart;
 
-    auto hologramFrame = ci->GetNextHologramFrame(timestamp);
+    // Convert offset to microseconds.
+    LONGLONG offset = (LONGLONG)(msOffset * 1000.0f);
+
+    auto hologramFrame = ci->GetNextHologramFrame(timestamp - offset);
     if (hologramFrame != nullptr)
     {
         hologramFrame->rotX = rotX;
@@ -240,11 +243,9 @@ UNITYDLL void UpdateSpectatorView()
     if (cachedTime != colorTime)
     {
         LONGLONG frameDuration = ci->GetColorDuration();
-        float buffer = 0.1f * frameDuration;
-
         // Find a pose on the leading end of this color frame.
         const auto frame = ci->FindClosestHologramFrame(
-            colorTime - frameDuration + buffer, (LONGLONG)(_frameOffset * (float)frameDuration));
+            colorTime - frameDuration, (LONGLONG)(_frameOffset * (float)frameDuration));
 
         if (frame != nullptr)
         {

SpectatorView/Samples/SharedHolograms/Assets/Addons/HolographicCameraRig/SV_UNET/Scripts/PlayerController.cs

@@ -26,7 +26,7 @@ private static extern bool GetHeadTransform(System.IntPtr unityCoordinateSystem,
         [DllImport("UnityCompositorInterface")]
         private static extern bool SetHologramPose(
             float rotX,  float rotY,  float rotZ,  float rotW,
-            float posX,  float posY,  float posZ);
+            float posX,  float posY,  float posZ, float msOffset);
 #endif
         #endregion
 
@@ -40,7 +40,6 @@ public override float GetNetworkSendInterval()
             return 0.033f;
         }
 
-
         private static PlayerController _Instance = null;
         public static PlayerController Instance
         {
@@ -73,6 +72,9 @@ public bool CanEstablishAnchor()
         [SyncVar]
         private Vector3 localPosition;
 
+        [SyncVar]
+        int spectatorViewPing = 0;
+
         /// <summary>
         /// The rotation relative to the shared world anchor.
         /// </summary>
@@ -109,10 +111,15 @@ public void SetColorDuration(long value)
         /// <param name="postion">the localPosition to set</param>
         /// <param name="rotation">the localRotation to set</param>
         [Command(channel = 1)]
-        public void CmdTransform(Vector3 postion, Quaternion rotation)
+        public void CmdTransform(Vector3 postion, Quaternion rotation, int ping)
         {
             localPosition = postion;
             localRotation = rotation;
+
+            if (IsSV())
+            {
+                spectatorViewPing = ping;
+            }
         }
 
         [SyncVar(hook = "AnchorEstablishedChanged")]
@@ -612,7 +619,7 @@ private void Update()
                 if (IsSV())
                 {
                     SetHologramPose(localRotation.x, localRotation.y, localRotation.z, localRotation.w,
-                        localPosition.x, localPosition.y, localPosition.z);
+                        localPosition.x, localPosition.y, localPosition.z, (float)spectatorViewPing / 2.0f);
                 }
 #endif
                 return;
@@ -657,9 +664,15 @@ private void Update()
                 transform.rotation = Camera.main.transform.rotation;
             }
 
-            // For UNET we use a command to signal the host to update our local position
-            // and rotation
-            CmdTransform(transform.localPosition, transform.localRotation);
+            // For UNET we use a command to signal the host to update our local position and rotation
+            if (IsSV())
+            {
+                CmdTransform(transform.localPosition, transform.localRotation, NetworkManager.singleton.client.GetRTT());
+            }
+            else
+            {
+                CmdTransform(transform.localPosition, transform.localRotation, 0);
+            }
         }
 
         /// <summary>