diff --git a/Runtime/Scripts/AudioStream.cs b/Runtime/Scripts/AudioStream.cs index b0d185b6..941f1b20 100644 --- a/Runtime/Scripts/AudioStream.cs +++ b/Runtime/Scripts/AudioStream.cs @@ -96,8 +96,6 @@ private void OnAudioStreamEvent(AudioStreamEvent e) if (e.MessageCase != AudioStreamEvent.MessageOneofCase.FrameReceived) return; - var frame = new AudioFrame(e.FrameReceived.Frame); - lock (_lock) { if (_numChannels == 0) @@ -105,15 +103,50 @@ private void OnAudioStreamEvent(AudioStreamEvent e) unsafe { - var uFrame = _resampler.RemixAndResample(frame, _numChannels, _sampleRate); - if (uFrame != null) + // Change deal with issue in newer LiveKit where channels are returned incorrectly + // -https://github.com/livekit/client-sdk-unity/issues/169 + // (plus some new changes to reduce garbage creation) + if (e.FrameReceived.Frame.Info.NumChannels == 1 && _numChannels == 2) { - var data = new Span(uFrame.Data.ToPointer(), uFrame.Length); - _buffer?.Write(data); - } + int samplesPerChannel = (int)e.FrameReceived.Frame.Info.SamplesPerChannel; + int monoLengthBytes = + (int)(e.FrameReceived.Frame.Info.SamplesPerChannel * + e.FrameReceived.Frame.Info.NumChannels * + sizeof(short)); + + // Span over the incoming mono audio bytes + var monoByteSpan = new ReadOnlySpan( + (void*)e.FrameReceived.Frame.Info.DataPtr, + monoLengthBytes); + + // Treat them as 16-bit samples + var monoSamples = MemoryMarshal.Cast(monoByteSpan); + + // Allocate stereo buffer on the stack: 2 channels * samples * sizeof(short) + Span stereoSamples = stackalloc short[samplesPerChannel * 2]; + + for (int i = 0; i < samplesPerChannel; i++) + { + short sample = monoSamples[i]; // mono + int dstIndex = i * 2; + + stereoSamples[dstIndex] = sample; // Left + stereoSamples[dstIndex + 1] = sample; // Right + } + + // Cast the stereo short span to bytes and write to the buffer + _buffer?.Write(MemoryMarshal.AsBytes(stereoSamples)); + } + else + { + //TODO add support here if they fix the above bug + } } } + // Change - need to drop handle here because it would normally be done + // within AudioFrame, without memory will be leaked on the plugin side + NativeMethods.FfiDropHandle((IntPtr)e.FrameReceived.Frame.Handle.Id); } public void Dispose() @@ -126,8 +159,10 @@ private void Dispose(bool disposing) { if (!_disposed && disposing) { + FfiClient.Instance.AudioStreamEventReceived -= OnAudioStreamEvent; _audioSource.Stop(); UnityEngine.Object.Destroy(_audioSource.GetComponent()); + _buffer?.Dispose(); } _disposed = true; } diff --git a/Runtime/Scripts/Internal/FFIClient.cs b/Runtime/Scripts/Internal/FFIClient.cs index d037a3bb..ec450117 100644 --- a/Runtime/Scripts/Internal/FFIClient.cs +++ b/Runtime/Scripts/Internal/FFIClient.cs @@ -185,7 +185,7 @@ public void Release(FfiResponse response) ffiResponsePool.Release(response); } - public FfiResponse SendRequest(FfiRequest request) + public FfiResponse SendRequest(FfiRequest request, bool requiresResponse = true) { try { @@ -203,9 +203,18 @@ public FfiResponse SendRequest(FfiRequest request) out UIntPtr dataLen ); var dataSpan = new Span(dataPtr, (int)dataLen.ToUInt64()); - var response = responseParser.ParseFrom(dataSpan)!; - NativeMethods.FfiDropHandle(handle); - return response; + + if (requiresResponse) + { + var response = responseParser.ParseFrom(dataSpan)!; + NativeMethods.FfiDropHandle(handle); + return response; + } + else + { + NativeMethods.FfiDropHandle(handle); + return null; + } } } } diff --git a/Runtime/Scripts/Internal/FFIClients/IFFIClient.cs b/Runtime/Scripts/Internal/FFIClients/IFFIClient.cs index 4d0cdb38..2def9296 100644 --- a/Runtime/Scripts/Internal/FFIClients/IFFIClient.cs +++ b/Runtime/Scripts/Internal/FFIClients/IFFIClient.cs @@ -8,7 +8,7 @@ namespace LiveKit.Internal.FFIClients /// public interface IFFIClient : IDisposable { - FfiResponse SendRequest(FfiRequest request); + FfiResponse SendRequest(FfiRequest request, bool requireResponse = true); void Release(FfiResponse response); } diff --git a/Runtime/Scripts/Internal/FFIClients/Requests/FFIBridge.cs b/Runtime/Scripts/Internal/FFIClients/Requests/FFIBridge.cs index 2d3f1523..d9fba415 100644 --- a/Runtime/Scripts/Internal/FFIClients/Requests/FFIBridge.cs +++ b/Runtime/Scripts/Internal/FFIClients/Requests/FFIBridge.cs @@ -15,8 +15,8 @@ public class FFIBridge : IFFIBridge public static FFIBridge Instance => instance.Value; - private readonly IFFIClient ffiClient; - private readonly IMultiPool multiPool; + public readonly IFFIClient ffiClient; + public readonly IMultiPool multiPool; public FFIBridge(IFFIClient client, IMultiPool multiPool) { diff --git a/Runtime/Scripts/RtcAudioSource.cs b/Runtime/Scripts/RtcAudioSource.cs index ef512733..0d0de432 100644 --- a/Runtime/Scripts/RtcAudioSource.cs +++ b/Runtime/Scripts/RtcAudioSource.cs @@ -126,26 +126,33 @@ static short FloatToS16(float v) for (int i = 0; i < data.Length; i++) _frameData[i] = FloatToS16(data[i]); + //Change - hand to make FFIBridge properties public to do this which isn't great // Capture the frame. - using var request = FFIBridge.Instance.NewRequest(); - using var audioFrameBufferInfo = request.TempResource(); + CaptureAudioFrameRequest pushFrame = FFIBridge.Instance.multiPool.Get(); + AudioFrameBufferInfo audioFrameBufferInfo = FFIBridge.Instance.multiPool.Get(); + + FfiRequest ffiRequest = FFIBridge.Instance.multiPool.Get(); - var pushFrame = request.request; pushFrame.SourceHandle = (ulong)Handle.DangerousGetHandle(); pushFrame.Buffer = audioFrameBufferInfo; unsafe { - pushFrame.Buffer.DataPtr = (ulong)NativeArrayUnsafeUtility - .GetUnsafePtr(_frameData); + pushFrame.Buffer.DataPtr = (ulong)NativeArrayUnsafeUtility + .GetUnsafePtr(_frameData); } pushFrame.Buffer.NumChannels = (uint)channels; pushFrame.Buffer.SampleRate = (uint)sampleRate; pushFrame.Buffer.SamplesPerChannel = (uint)data.Length / (uint)channels; - using var response = request.Send(); - FfiResponse res = response; - // Wait for async callback, log an error if the capture fails. + ffiRequest.CaptureAudioFrame = pushFrame; + FFIBridge.Instance.ffiClient.SendRequest(ffiRequest, false); + + + // Changes - this was creating memory because of the callback being created, + // might be able to make a call back and cache the asyncID to get around that problem + // TODO: Investigate further and optimize if needed + /* var asyncId = res.CaptureAudioFrame.AsyncId; void Callback(CaptureAudioFrameCallback callback) { @@ -155,6 +162,17 @@ void Callback(CaptureAudioFrameCallback callback) FfiClient.Instance.CaptureAudioFrameReceived -= Callback; } FfiClient.Instance.CaptureAudioFrameReceived += Callback; + */ + + ffiRequest.CaptureAudioFrame = null; + pushFrame.Buffer.ClearDataPtr(); + pushFrame.Buffer = null; + ffiRequest.ClearMessage(); + + + FFIBridge.Instance.multiPool.Release(ffiRequest); + FFIBridge.Instance.multiPool.Release(pushFrame); + FFIBridge.Instance.multiPool.Release(audioFrameBufferInfo); } ///