Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -5,3 +5,4 @@ riderModule.iml
/_ReSharper.Caches/
/.vs
/output
/.idea
5 changes: 3 additions & 2 deletions VRCFTPicoModule/Data/BlendShapeIndex.cs
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
namespace VRCFTPicoModule.Data;
// ReSharper disable InconsistentNaming
namespace VRCFTPicoModule.Data;

public class BlendShape
public abstract class BlendShape
{
public enum Index
{
Expand Down
2 changes: 1 addition & 1 deletion VRCFTPicoModule/Data/DataPacket.cs
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

namespace VRCFTPicoModule.Data
{
public class DataPacket
public abstract class DataPacket
{
[StructLayout(LayoutKind.Sequential, Pack = 1)]
public struct DataPackHeader
Expand Down
2 changes: 1 addition & 1 deletion VRCFTPicoModule/Data/LegacyDataPacket.cs
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

namespace VRCFTPicoModule.Data
{
public class LegacyDataPacket
public abstract class LegacyDataPacket
{
[StructLayout(LayoutKind.Sequential, Pack = 1)]
public struct DataPackBody
Expand Down
96 changes: 55 additions & 41 deletions VRCFTPicoModule/Utils/Updater.cs
Original file line number Diff line number Diff line change
Expand Up @@ -6,52 +6,66 @@
using VRCFaceTracking.Core.Library;
using VRCFaceTracking.Core.Params.Expressions;
using VRCFTPicoModule.Data;
using VRCFTPicoModule.Utils;

namespace VRCFTPicoModule
namespace VRCFTPicoModule.Utils
{
public class Updater
public class Updater()
{
private readonly UdpClient udpClient;
private readonly ILogger logger;
private int timeOut = 0;
private float lastMouthLeft = 0f;
private float lastMouthRight = 0f;
private const float smoothingFactor = 0.5f;
private bool isLegecy = false;
public ModuleState moduleState;

public Updater(UdpClient udpClient, ILogger logger, bool isLegecy)
private readonly UdpClient? _udpClient;
private readonly ILogger? _logger;
private readonly bool _isLegacy;
private readonly (bool, bool) _trackingAvailable;

public Updater(UdpClient udpClient, ILogger logger, bool isLegacy, (bool, bool) trackingAvailable) : this()
{
this.udpClient = udpClient;
this.logger = logger;
this.isLegecy = isLegecy;
_udpClient = udpClient;
_logger = logger;
_isLegacy = isLegacy;
_trackingAvailable = trackingAvailable;
}

public void Update()

private int _timeOut;
private float _lastMouthLeft;
private float _lastMouthRight;
private const float SmoothingFactor = 0.5f;
private ModuleState _moduleState;

public void Update(ModuleState state)
{
if (moduleState != ModuleState.Active) return;
if (_udpClient == null)
return;

if (_logger == null)
return;

_udpClient.Client.ReceiveTimeout = 100;
_moduleState = state;

if (_moduleState != ModuleState.Active) return;

try
{
var endPoint = new IPEndPoint(IPAddress.Any, 0);
var data = udpClient.Receive(ref endPoint);
var pShape = ParseData(data, isLegecy);

UpdateEye(pShape);
UpdateExpression(pShape);
var data = _udpClient.Receive(ref endPoint);
var pShape = ParseData(data, _isLegacy);

if (_trackingAvailable.Item1)
UpdateEye(pShape);

if (_trackingAvailable.Item2)
UpdateExpression(pShape);
}
catch (SocketException ex) when (ex.SocketErrorCode == SocketError.TimedOut)
{
if (++timeOut > 600)
if (++_timeOut > 600)
{
logger.LogWarning("Receive data timed out.");
timeOut = 0;
_logger.LogWarning("Receive data timed out.");
_timeOut = 0;
}
}
catch (Exception ex)
{
logger.LogWarning("Update failed with exception: {0}", ex);
_logger.LogWarning("Update failed with exception: {0}", ex);
}
}

Expand All @@ -62,11 +76,11 @@ private static float[] ParseData(byte[] data, bool isLegacy)

if (data.Length >= Marshal.SizeOf<DataPacket.DataPackHeader>() + Marshal.SizeOf<DataPacket.DataPackBody>())
{
var header = DataPacketHelpers.ByteArrayToStructure<DataPacket.DataPackHeader>(data, 0);
var header = DataPacketHelpers.ByteArrayToStructure<DataPacket.DataPackHeader>(data);
if (header.trackingType == 2)
return DataPacketHelpers.ByteArrayToStructure<DataPacket.DataPackBody>(data, Marshal.SizeOf<DataPacket.DataPackHeader>()).blendShapeWeight;
}
return Array.Empty<float>();
return [];
}

private static void UpdateEye(float[] pShape)
Expand All @@ -84,10 +98,7 @@ private static void UpdateEye(float[] pShape)
eye.Right.Gaze.x = pShape[(int)BlendShape.Index.EyeLookOut_R] - pShape[(int)BlendShape.Index.EyeLookIn_R];
eye.Right.Gaze.y = pShape[(int)BlendShape.Index.EyeLookUp_R] - pShape[(int)BlendShape.Index.EyeLookDown_R];
#endregion
}

private void UpdateExpression(float[] pShape)
{

#region Brow
SetParam(pShape, BlendShape.Index.BrowInnerUp, UnifiedExpressions.BrowInnerUpLeft);
SetParam(pShape, BlendShape.Index.BrowInnerUp, UnifiedExpressions.BrowInnerUpRight);
Expand All @@ -105,7 +116,10 @@ private void UpdateExpression(float[] pShape)
SetParam(pShape, BlendShape.Index.EyeWide_L, UnifiedExpressions.EyeWideLeft);
SetParam(pShape, BlendShape.Index.EyeWide_R, UnifiedExpressions.EyeWideRight);
#endregion
}

private void UpdateExpression(float[] pShape)
{
#region Jaw
SetParam(pShape, BlendShape.Index.JawOpen, UnifiedExpressions.JawOpen);
SetParam(pShape, BlendShape.Index.JawLeft, UnifiedExpressions.JawLeft);
Expand All @@ -118,11 +132,11 @@ private void UpdateExpression(float[] pShape)
SetParam(pShape, BlendShape.Index.CheekSquint_L, UnifiedExpressions.CheekSquintLeft);
SetParam(pShape, BlendShape.Index.CheekSquint_R, UnifiedExpressions.CheekSquintRight);

float mouthLeft = SmoothValue(pShape[(int)BlendShape.Index.MouthLeft], ref lastMouthLeft);
float mouthRight = SmoothValue(pShape[(int)BlendShape.Index.MouthRight], ref lastMouthRight);
var mouthLeft = SmoothValue(pShape[(int)BlendShape.Index.MouthLeft], ref _lastMouthLeft);
var mouthRight = SmoothValue(pShape[(int)BlendShape.Index.MouthRight], ref _lastMouthRight);

float cheekPuff = pShape[(int)BlendShape.Index.CheekPuff];
float diffThreshold = 0.1f;
var cheekPuff = pShape[(int)BlendShape.Index.CheekPuff];
const float diffThreshold = 0.1f;

if (cheekPuff > 0.1f)
{
Expand Down Expand Up @@ -195,13 +209,13 @@ private void UpdateExpression(float[] pShape)
#endregion

#region Tongue
SetParam(pShape, BlendShape.Index.TongueOut, UnifiedExpressions.TongueOut);
SetParam(pShape[(int)BlendShape.Index.TongueOut] > 0f ? 1f : 0f, UnifiedExpressions.TongueOut);
#endregion
}

private float SmoothValue(float newValue, ref float lastValue)
private static float SmoothValue(float newValue, ref float lastValue)
{
lastValue += (newValue - lastValue) * smoothingFactor;
lastValue += (newValue - lastValue) * SmoothingFactor;
return lastValue;
}

Expand Down
63 changes: 38 additions & 25 deletions VRCFTPicoModule/VRCFTPicoModule.cs
Original file line number Diff line number Diff line change
@@ -1,79 +1,92 @@
using Microsoft.Extensions.Logging;
using System.Net.Sockets;
using VRCFaceTracking;
using VRCFTPicoModule.Utils;

namespace VRCFTPicoModule;

public partial class VRCFTPicoModule : ExtTrackingModule
public class VRCFTPicoModule : ExtTrackingModule
{
private static readonly int[] Ports = { 29765, 29763 };
private static readonly int[] Ports = [29765, 29763];
private static readonly UdpClient[] Clients = Ports.Select(port => new UdpClient(port) { Client = { ReceiveTimeout = 100 } }).ToArray();
private static UdpClient udpClient = new();
private static int Port = 0;
private Updater? updater;
private static UdpClient _udpClient = new();
private static int _port;
private Updater? _updater;
private (bool, bool) _trackingAvailable;

public override (bool SupportsEye, bool SupportsExpression) Supported => (true, true);

public override (bool eyeSuccess, bool expressionSuccess) Initialize(bool eyeAvailable, bool expressionAvailable)
{
Logger.LogInformation("Starting initialization");
_trackingAvailable = (eyeAvailable, expressionAvailable);
var initializationResult = InitializeAsync().GetAwaiter().GetResult();
if (initializationResult.eyeSuccess && initializationResult.expressionSuccess)
{

if (!initializationResult.Item1 || !initializationResult.Item2)
UpdateModuleInfo();
}

return initializationResult;
}

private async Task<(bool eyeSuccess, bool expressionSuccess)> InitializeAsync()
{
Logger.LogDebug("Initializing UDP Clients on ports: {0}", string.Join(", ", Ports));

int portIndex = await ListenOnPorts();
var portIndex = await ListenOnPorts();
if (portIndex == -1) return (false, false);

Port = Ports[portIndex];
udpClient = new UdpClient(Port);
Logger.LogInformation("Using port: {0}", Port);
_port = Ports[portIndex];
_udpClient = new UdpClient(_port);
Logger.LogInformation("Using port: {0}", _port);

if (!_trackingAvailable.Item1)
Logger.LogInformation("Eye tracking is disabled");
if (!_trackingAvailable.Item2)
Logger.LogInformation("Expression tracking is disabled");

updater = new Updater(udpClient, Logger, Port == Ports[1]);
_updater = new Updater(_udpClient, Logger, _port == Ports[1], _trackingAvailable);

return (true, true);
return _trackingAvailable;
}

private void UpdateModuleInfo()
{
ModuleInformation.Name = "PICO Connect";
var stream = GetType().Assembly.GetManifestResourceStream("VRCFTPicoModule.Assets.pico.png");
ModuleInformation.StaticImages = stream != null ? new List<Stream> { stream } : ModuleInformation.StaticImages;
ModuleInformation.StaticImages = stream != null ? [stream] : ModuleInformation.StaticImages;
}

private async Task<int> ListenOnPorts()
{
try
{
var tasks = Clients.Select(client => client.ReceiveAsync()).ToArray();
var completedTask = await Task.WhenAny(tasks);

if (completedTask != null)

if (tasks.Length == 0)
{
foreach (var client in Clients) client.Dispose();
return Array.IndexOf(tasks, completedTask);
return -1;
}

var completedTask = await Task.WhenAny(tasks);

foreach (var client in Clients) client.Dispose();

return Array.IndexOf(tasks, completedTask);
}
catch (Exception ex)
{
Logger.LogError("Initialization failed, exception: {0}", ex);
}

return -1;
}

public override void Update()
{
if (updater == null)
if (_updater == null)
return;
updater.moduleState = Status;
updater.Update();

_updater.Update(Status);
}

public override void Teardown()
Expand All @@ -82,7 +95,7 @@ public override void Teardown()
{
client.Dispose();
}
udpClient.Dispose();
updater = null;
_udpClient.Dispose();
_updater = null;
}
}
2 changes: 1 addition & 1 deletion VRCFTPicoModule/VRCFTPicoModule.csproj
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net7.0</TargetFramework>
<LangVersion>latest</LangVersion>
<LangVersion>preview</LangVersion>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
<AssemblyVersion>0.1.6</AssemblyVersion>
Expand Down