Skip to content

Commit 0e08a97

Browse files
fieldsJacksonGfieldsJacksonG
authored
Merge pull request microsoft#144 from fieldsJacksonG/master
Update spectator view to the latest HoloToolkit (1.5.7)
2 parents 91e7423 + 5c9e832 commit 0e08a97

File tree

473 files changed

+60455
-55305
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

473 files changed

+60455
-55305
lines changed

‎SpectatorView/Calibration/Calibration/CalibrationApp.cpp

+1-1
Original file line numberDiff line numberDiff line change
@@ -100,7 +100,7 @@ void CalibrationApp::Initialize(HWND window, int width, int height)
100100
frameProvider = new DeckLinkManager(true, true);
101101
#endif
102102
#if USE_OPENCV
103-
frameProvider = new OpenCVFrameProvider();
103+
frameProvider = new OpenCVFrameProvider(false);
104104
#endif
105105

106106
frameProvider->Initialize(srv, nullptr);

‎SpectatorView/Compositor/CompositorDLL/CompositorInterface.cpp

+2-2
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,7 @@ bool CompositorInterface::Initialize(ID3D11Device* device, ID3D11ShaderResourceV
5050

5151
_device = device;
5252

53-
hologramQueue = new HologramQueue(device);
53+
hologramQueue = new HologramQueue();
5454

5555
return SUCCEEDED(frameProvider->Initialize(colorSRV, outputTexture));
5656
}
@@ -124,7 +124,7 @@ void CompositorInterface::StopFrameProvider()
124124
#endif
125125
}
126126

127-
LONGLONG CompositorInterface::GetColorTime()
127+
LONGLONG CompositorInterface::GetTimestamp()
128128
{
129129
if (frameProvider != nullptr)
130130
{

‎SpectatorView/Compositor/CompositorDLL/CompositorInterface.h

+2-1
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,8 @@ class CompositorInterface
5757
DLLEXPORT void Update();
5858
DLLEXPORT void StopFrameProvider();
5959

60-
DLLEXPORT LONGLONG GetColorTime();
60+
DLLEXPORT LONGLONG GetTimestamp();
61+
6162
DLLEXPORT LONGLONG GetColorDuration();
6263

6364
DLLEXPORT void TakePicture(ID3D11Device* device, int width, int height, int bpp,

‎SpectatorView/Compositor/CompositorDLL/DeckLinkDevice.cpp

+27-10
Original file line numberDiff line numberDiff line change
@@ -79,9 +79,10 @@ DeckLinkDevice::~DeckLinkDevice()
7979
DeleteCriticalSection(&m_outputCriticalSection);
8080
DeleteCriticalSection(&m_frameAccessCriticalSection);
8181

82-
delete[] cachedBuffer;
83-
delete[] stagingBuffer;
82+
delete[] thirdCachedBuffer;
83+
delete[] secondCachedBuffer;
8484
delete[] latestBuffer;
85+
delete[] stagingBuffer;
8586
delete[] outputBuffer;
8687
delete[] outputBufferRaw;
8788
}
@@ -148,7 +149,8 @@ bool DeckLinkDevice::Init(ID3D11ShaderResourceView* colorSRV, ID3D11Texture2D* o
148149
BSTR deviceNameBSTR = NULL;
149150

150151
ZeroMemory(rawBuffer, FRAME_BUFSIZE_RAW);
151-
ZeroMemory(cachedBuffer, FRAME_BUFSIZE);
152+
ZeroMemory(thirdCachedBuffer, FRAME_BUFSIZE);
153+
ZeroMemory(secondCachedBuffer, FRAME_BUFSIZE);
152154
ZeroMemory(latestBuffer, FRAME_BUFSIZE);
153155
ZeroMemory(outputBuffer, FRAME_BUFSIZE);
154156
ZeroMemory(outputBufferRaw, FRAME_BUFSIZE_RAW);
@@ -372,13 +374,16 @@ HRESULT DeckLinkDevice::VideoInputFrameArrived(/* in */ IDeckLinkVideoInputFrame
372374
{
373375
if (frame->GetBytes((void**)&rawBuffer) == S_OK)
374376
{
377+
// Always return the latest buffer when using the CPU.
375378
if (_useCPU)
376379
{
377-
DirectXHelper::ConvertYUVtoBGRA(rawBuffer, cachedBuffer, FRAME_WIDTH, FRAME_HEIGHT, true);
380+
DirectXHelper::ConvertYUVtoBGRA(rawBuffer, latestBuffer, FRAME_WIDTH, FRAME_HEIGHT, true);
378381
}
379382
else
380383
{
381-
memcpy(cachedBuffer, rawBuffer, FRAME_BUFSIZE_RAW);
384+
memcpy(thirdCachedBuffer, secondCachedBuffer, FRAME_BUFSIZE_RAW);
385+
memcpy(secondCachedBuffer, latestBuffer, FRAME_BUFSIZE_RAW);
386+
memcpy(latestBuffer, rawBuffer, FRAME_BUFSIZE_RAW);
382387
}
383388
}
384389
}
@@ -391,11 +396,14 @@ HRESULT DeckLinkDevice::VideoInputFrameArrived(/* in */ IDeckLinkVideoInputFrame
391396
//TODO: Remove this block if R and B components are swapped in color feed.
392397
memcpy(stagingBuffer, localFrameBuffer, FRAME_BUFSIZE);
393398
DirectXHelper::ConvertBGRAtoRGBA(stagingBuffer, FRAME_WIDTH, FRAME_HEIGHT, true);
394-
memcpy(cachedBuffer, stagingBuffer, FRAME_BUFSIZE);
399+
// Do not cache frames when using the CPU
400+
memcpy(latestBuffer, stagingBuffer, FRAME_BUFSIZE);
395401
}
396402
else
397403
{
398-
memcpy(cachedBuffer, localFrameBuffer, FRAME_BUFSIZE);
404+
memcpy(thirdCachedBuffer, secondCachedBuffer, FRAME_BUFSIZE);
405+
memcpy(secondCachedBuffer, latestBuffer, FRAME_BUFSIZE);
406+
memcpy(latestBuffer, localFrameBuffer, FRAME_BUFSIZE);
399407
}
400408
}
401409
}
@@ -404,7 +412,9 @@ HRESULT DeckLinkDevice::VideoInputFrameArrived(/* in */ IDeckLinkVideoInputFrame
404412
frame->GetStreamTime(&t, &frameDuration, QPC_MULTIPLIER);
405413

406414
// Get frame time.
407-
cachedTimeStamp = time.QuadPart;
415+
thirdTimeStamp = secondTimeStamp;
416+
secondTimeStamp = latestTimeStamp;
417+
latestTimeStamp = time.QuadPart;
408418

409419
dirtyFrame = false;
410420

@@ -445,13 +455,20 @@ HRESULT DeckLinkDevice::VideoInputFrameArrived(/* in */ IDeckLinkVideoInputFrame
445455
void DeckLinkDevice::Update()
446456
{
447457
if (_colorSRV != nullptr &&
448-
cachedBuffer != nullptr &&
449458
device != nullptr)
450459
{
451460
if (!dirtyFrame)
452461
{
453462
dirtyFrame = true;
454-
DirectXHelper::UpdateSRV(device, _colorSRV, cachedBuffer, FRAME_WIDTH * FRAME_BPP);
463+
if (_useCPU && latestBuffer != nullptr)
464+
{
465+
// Do not cache when using CPU.
466+
DirectXHelper::UpdateSRV(device, _colorSRV, latestBuffer, FRAME_WIDTH * FRAME_BPP);
467+
}
468+
else if (!_useCPU && thirdCachedBuffer != nullptr)
469+
{
470+
DirectXHelper::UpdateSRV(device, _colorSRV, thirdCachedBuffer, FRAME_WIDTH * FRAME_BPP);
471+
}
455472

456473
EnterCriticalSection(&m_frameAccessCriticalSection);
457474
isVideoFrameReady = true;

‎SpectatorView/Compositor/CompositorDLL/DeckLinkDevice.h

+11-9
Original file line numberDiff line numberDiff line change
@@ -57,18 +57,20 @@ class DeckLinkDevice : public IDeckLinkInputCallback
5757
CRITICAL_SECTION m_outputCriticalSection;
5858

5959
BYTE* localFrameBuffer;
60-
BYTE* rawBuffer = new BYTE[FRAME_BUFSIZE_RAW];
60+
BYTE* rawBuffer = new BYTE[FRAME_BUFSIZE_RAW];
6161

62-
BYTE* cachedBuffer = new BYTE[FRAME_BUFSIZE];
63-
BYTE* stagingBuffer = new BYTE[FRAME_BUFSIZE];
64-
BYTE* latestBuffer = new BYTE[FRAME_BUFSIZE];
65-
BYTE* outputBuffer = new BYTE[FRAME_BUFSIZE];
66-
BYTE* outputBufferRaw = new BYTE[FRAME_BUFSIZE_RAW];
62+
BYTE* thirdCachedBuffer = new BYTE[FRAME_BUFSIZE];
63+
BYTE* secondCachedBuffer = new BYTE[FRAME_BUFSIZE];
64+
BYTE* latestBuffer = new BYTE[FRAME_BUFSIZE];
65+
BYTE* stagingBuffer = new BYTE[FRAME_BUFSIZE];
66+
BYTE* outputBuffer = new BYTE[FRAME_BUFSIZE];
67+
BYTE* outputBufferRaw = new BYTE[FRAME_BUFSIZE_RAW];
6768

6869
BMDTimeValue frameDuration = 0;
6970

7071
LONGLONG latestTimeStamp = 0;
71-
LONGLONG cachedTimeStamp = 0;
72+
LONGLONG secondTimeStamp = 0;
73+
LONGLONG thirdTimeStamp = 0;
7274

7375
bool dirtyFrame = true;
7476
bool isVideoFrameReady = false;
@@ -104,9 +106,9 @@ class DeckLinkDevice : public IDeckLinkInputCallback
104106
virtual HRESULT STDMETHODCALLTYPE VideoInputFormatChanged (/* in */ BMDVideoInputFormatChangedEvents notificationEvents, /* in */ IDeckLinkDisplayMode *newDisplayMode, /* in */ BMDDetectedVideoInputFormatFlags detectedSignalFlags);
105107
virtual HRESULT STDMETHODCALLTYPE VideoInputFrameArrived (/* in */ IDeckLinkVideoInputFrame* frame, /* in */ IDeckLinkAudioInputPacket* audioPacket);
106108

107-
LONGLONG GetTimeStamp()
109+
LONGLONG GetTimestamp()
108110
{
109-
return cachedTimeStamp;
111+
return thirdTimeStamp;
110112
}
111113

112114
LONGLONG GetDurationHNS()

‎SpectatorView/Compositor/CompositorDLL/DeckLinkManager.cpp

+1-1
Original file line numberDiff line numberDiff line change
@@ -123,7 +123,7 @@ LONGLONG DeckLinkManager::GetTimestamp()
123123
{
124124
if (IsEnabled())
125125
{
126-
return deckLinkDevice->GetTimeStamp();
126+
return deckLinkDevice->GetTimestamp();
127127
}
128128

129129
return 0;

‎SpectatorView/Compositor/CompositorDLL/DeckLinkManager.h

+3
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,10 @@ class DeckLinkManager : public IFrameProvider
1616

1717
HRESULT Initialize(ID3D11ShaderResourceView* colorSRV, ID3D11Texture2D* outputTexture);
1818

19+
// 3 frames are caches for reliable hologram stability:
20+
// Get the timestamp of the earliest (and currently rendered) cached frame.
1921
LONGLONG GetTimestamp();
22+
2023
LONGLONG GetDurationHNS();
2124

2225
void Update();

‎SpectatorView/Compositor/CompositorDLL/ElgatoFrameProvider.h

-2
Original file line numberDiff line numberDiff line change
@@ -106,8 +106,6 @@ class ElgatoFrameProvider : public IFrameProvider
106106
IBaseFilter *pNullF = NULL;
107107
ElgatoSampleCallback *frameCallback = NULL;
108108
IElgatoVideoCaptureFilter6 *filter = NULL;
109-
110-
BYTE* cachedBuffer = new BYTE[FRAME_BUFSIZE];
111109
};
112110

113111
#endif

‎SpectatorView/Compositor/CompositorDLL/ElgatoSampleCallback.cpp

+10-4
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,9 @@ STDMETHODIMP ElgatoSampleCallback::BufferCB(double time, BYTE *pBuffer, long len
2020
// Get frame time.
2121
LARGE_INTEGER t;
2222
QueryPerformanceCounter(&t);
23-
cachedTimestamp = t.QuadPart;
23+
thirdTimeStamp = secondTimeStamp;
24+
secondTimeStamp = latestTimeStamp;
25+
latestTimeStamp = t.QuadPart;
2426

2527
int copyLength = length;
2628
if (copyLength > FRAME_BUFSIZE)
@@ -29,7 +31,10 @@ STDMETHODIMP ElgatoSampleCallback::BufferCB(double time, BYTE *pBuffer, long len
2931
copyLength = FRAME_BUFSIZE;
3032
}
3133

32-
memcpy(cachedBytes, pBuffer, copyLength);
34+
memcpy(thirdCachedBuffer, secondCachedBuffer, FRAME_BUFSIZE);
35+
memcpy(secondCachedBuffer, latestBuffer, FRAME_BUFSIZE);
36+
memcpy(latestBuffer, pBuffer, copyLength);
37+
3338
EnterCriticalSection(&frameAccessCriticalSection);
3439
isVideoFrameReady = true;
3540
LeaveCriticalSection(&frameAccessCriticalSection);
@@ -42,13 +47,14 @@ void ElgatoSampleCallback::UpdateSRV(ID3D11ShaderResourceView* srv, bool useCPU)
4247
if (useCPU)
4348
{
4449
BYTE* stagingBytes = new BYTE[FRAME_BUFSIZE];
45-
DirectXHelper::ConvertYUVtoBGRA(cachedBytes, stagingBytes, FRAME_WIDTH, FRAME_HEIGHT, true);
50+
// Do not cache when using the CPU
51+
DirectXHelper::ConvertYUVtoBGRA(latestBuffer, stagingBytes, FRAME_WIDTH, FRAME_HEIGHT, true);
4652
DirectXHelper::UpdateSRV(_device, srv, stagingBytes, FRAME_WIDTH * FRAME_BPP);
4753
delete[] stagingBytes;
4854
}
4955
else
5056
{
51-
DirectXHelper::UpdateSRV(_device, srv, cachedBytes, FRAME_WIDTH * FRAME_BPP);
57+
DirectXHelper::UpdateSRV(_device, srv, thirdCachedBuffer, FRAME_WIDTH * FRAME_BPP);
5258
}
5359
}
5460

‎SpectatorView/Compositor/CompositorDLL/ElgatoSampleCallback.h

+8-3
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,7 @@ class ElgatoSampleCallback : public ISampleGrabberCB
5656

5757
LONGLONG GetTimestamp()
5858
{
59-
return cachedTimestamp;
59+
return thirdTimeStamp;
6060
}
6161

6262
bool IsVideoFrameReady();
@@ -65,8 +65,13 @@ class ElgatoSampleCallback : public ISampleGrabberCB
6565
ULONG m_cRef = 0;
6666

6767
ID3D11Device* _device;
68-
BYTE* cachedBytes = new BYTE[FRAME_BUFSIZE];
69-
LONGLONG cachedTimestamp = -1;
68+
BYTE* thirdCachedBuffer = new BYTE[FRAME_BUFSIZE];
69+
BYTE* secondCachedBuffer = new BYTE[FRAME_BUFSIZE];
70+
BYTE* latestBuffer = new BYTE[FRAME_BUFSIZE];
71+
72+
LONGLONG latestTimeStamp = 0;
73+
LONGLONG secondTimeStamp = 0;
74+
LONGLONG thirdTimeStamp = 0;
7075

7176
CRITICAL_SECTION frameAccessCriticalSection;
7277
bool isVideoFrameReady = false;

‎SpectatorView/Compositor/CompositorDLL/HologramQueue.cpp

+1-11
Original file line numberDiff line numberDiff line change
@@ -6,16 +6,12 @@
66

77
// These methods must all be called from the same thread.
88
// Otherwise, a lock will need to be added which will substantially slow down the render thread.
9-
HologramQueue::HologramQueue(ID3D11Device* device)
9+
HologramQueue::HologramQueue()
1010
{
1111
for (int i = 0; i < MAX_QUEUE_SIZE; i++)
1212
{
1313
m_holographicFrameQueue[i].timeStamp = INVALID_TIMESTAMP;
1414
m_holographicFrameQueue[i].m_id = i;
15-
if (m_holographicFrameQueue[i].holoTexture == nullptr)
16-
{
17-
m_holographicFrameQueue[i].holoTexture = DirectXHelper::CreateTexture(device, hologramQueueFrameData, HOLOGRAM_WIDTH, HOLOGRAM_HEIGHT, 4);
18-
}
1915
}
2016
}
2117

@@ -38,12 +34,6 @@ FrameMessage* HologramQueue::FindClosestFrame(LONGLONG timeStamp, LONGLONG frame
3834
for (int i = 0; i < MAX_QUEUE_SIZE; i++)
3935
{
4036
LONGLONG frameTime = m_holographicFrameQueue[i].timeStamp;
41-
42-
// Absolute value of timestamps in case QPC is currently reporting negative values.
43-
if (timeStamp < 0) { timeStamp *= -1; }
44-
if (frameTime < 0) { frameTime *= -1; }
45-
if (frameOffset < 0) { frameOffset *= -1; }
46-
4737
LONGLONG delta = timeStamp - frameTime - frameOffset;
4838

4939
if (delta >= 0 && delta < smallestDelta)

‎SpectatorView/Compositor/CompositorDLL/HologramQueue.h

+3-4
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99
#include <memory>
1010
#include <array>
1111

12-
#define MAX_QUEUE_SIZE 30
12+
#define MAX_QUEUE_SIZE 90
1313

1414
#define INVALID_TIMESTAMP -1
1515

@@ -20,8 +20,7 @@ typedef struct
2020
friend class HologramQueue;
2121
public:
2222
LONGLONG timeStamp = INVALID_TIMESTAMP;
23-
24-
ID3D11Texture2D* holoTexture;
23+
float rotX, rotY, rotZ, rotW, posX, posY, posZ;
2524

2625
int GetId() const { return m_id; }
2726
private:
@@ -34,7 +33,7 @@ static BYTE* hologramQueueFrameData = new BYTE[HOLOGRAM_BUFSIZE];
3433
class HologramQueue
3534
{
3635
public:
37-
HologramQueue(ID3D11Device* device);
36+
HologramQueue();
3837

3938
FrameMessage* GetNextFrame(LONGLONG timeStamp);
4039
FrameMessage* FindClosestFrame(LONGLONG timeStamp, LONGLONG frameOffset);

‎SpectatorView/Compositor/CompositorDLL/IFrameProvider.h

+3-1
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,9 @@ class IFrameProvider
99
public:
1010
// Set up the FrameProvider to start delivering frames.
1111
virtual HRESULT Initialize(ID3D11ShaderResourceView* colorSRV, ID3D11Texture2D* outputTexture) = 0;
12-
// Get the timestamp of the latest frame.
12+
13+
// 4 frames are caches for reliable hologram stability:
14+
// Get the timestamp of the currently rendered cached frame.
1315
virtual LONGLONG GetTimestamp() = 0;
1416

1517
virtual LONGLONG GetDurationHNS() = 0;

‎SpectatorView/Compositor/CompositorDLL/OpenCVFrameProvider.cpp

+17-5
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,8 @@
66

77
#if USE_OPENCV
88

9-
OpenCVFrameProvider::OpenCVFrameProvider()
9+
OpenCVFrameProvider::OpenCVFrameProvider(bool cacheFrames) :
10+
_cacheFrames(cacheFrames)
1011
{
1112
}
1213

@@ -65,7 +66,9 @@ void OpenCVFrameProvider::Update()
6566

6667
if (videoCapture->grab())
6768
{
68-
cachedTimestamp = time.QuadPart;
69+
thirdTimeStamp = secondTimeStamp;
70+
secondTimeStamp = latestTimeStamp;
71+
latestTimeStamp = time.QuadPart;
6972

7073
concurrency::create_task([=]
7174
{
@@ -94,7 +97,9 @@ void OpenCVFrameProvider::Update()
9497
EnterCriticalSection(&lock);
9598
dirtyFrame = false;
9699

97-
memcpy(cachedFrame, tmpData, FRAME_BUFSIZE);
100+
memcpy(thirdCachedBuffer, secondCachedBuffer, FRAME_BUFSIZE);
101+
memcpy(secondCachedBuffer, latestBuffer, FRAME_BUFSIZE);
102+
memcpy(latestBuffer, tmpData, FRAME_BUFSIZE);
98103

99104
LeaveCriticalSection(&lock);
100105
});
@@ -104,7 +109,14 @@ void OpenCVFrameProvider::Update()
104109
EnterCriticalSection(&lock);
105110
if (!dirtyFrame)
106111
{
107-
DirectXHelper::UpdateSRV(_device, _colorSRV, cachedFrame, FRAME_WIDTH * FRAME_BPP);
112+
if (!_cacheFrames && latestBuffer != nullptr)
113+
{
114+
DirectXHelper::UpdateSRV(_device, _colorSRV, latestBuffer, FRAME_WIDTH * FRAME_BPP);
115+
}
116+
else if (_cacheFrames && thirdCachedBuffer != nullptr)
117+
{
118+
DirectXHelper::UpdateSRV(_device, _colorSRV, thirdCachedBuffer, FRAME_WIDTH * FRAME_BPP);
119+
}
108120
dirtyFrame = true;
109121
}
110122
LeaveCriticalSection(&lock);
@@ -131,7 +143,7 @@ bool OpenCVFrameProvider::IsVideoFrameReady()
131143

132144
LONGLONG OpenCVFrameProvider::GetTimestamp()
133145
{
134-
return cachedTimestamp;
146+
return thirdTimeStamp;
135147
}
136148

137149
LONGLONG OpenCVFrameProvider::GetDurationHNS()

0 commit comments

Comments
 (0)