From 5d5493f0030871e376d802efafb8f69bda66b2ba Mon Sep 17 00:00:00 2001 From: cloudwebrtc Date: Thu, 8 Jan 2026 11:46:05 +0800 Subject: [PATCH 1/3] fix: fix memory leaks for audio stream. --- Runtime/Scripts/AudioStream.cs | 47 +++++++++++++++++-- Runtime/Scripts/Internal/FFIClient.cs | 17 +++++-- .../Scripts/Internal/FFIClients/IFFIClient.cs | 2 +- .../Internal/FFIClients/Requests/FFIBridge.cs | 4 +- Runtime/Scripts/RtcAudioSource.cs | 38 +++++++++++---- 5 files changed, 88 insertions(+), 20 deletions(-) diff --git a/Runtime/Scripts/AudioStream.cs b/Runtime/Scripts/AudioStream.cs index b0d185b6..4da2cbdf 100644 --- a/Runtime/Scripts/AudioStream.cs +++ b/Runtime/Scripts/AudioStream.cs @@ -96,7 +96,7 @@ private void OnAudioStreamEvent(AudioStreamEvent e) if (e.MessageCase != AudioStreamEvent.MessageOneofCase.FrameReceived) return; - var frame = new AudioFrame(e.FrameReceived.Frame); + //var frame = new AudioFrame(e.FrameReceived.Frame); lock (_lock) { @@ -105,15 +105,52 @@ private void OnAudioStreamEvent(AudioStreamEvent e) unsafe { - var uFrame = _resampler.RemixAndResample(frame, _numChannels, _sampleRate); - if (uFrame != null) + // Change deal with issue in newer LiveKit where channels are returned incorrectly + // -https://github.com/livekit/client-sdk-unity/issues/169 + // (plus some new changes to reduce garbage creation) + if (e.FrameReceived.Frame.Info.NumChannels == 1 && _numChannels == 2) { - var data = new Span(uFrame.Data.ToPointer(), uFrame.Length); - _buffer?.Write(data); + + int samplesPerChannel = (int)e.FrameReceived.Frame.Info.SamplesPerChannel; + int monoLengthBytes = + (int)(e.FrameReceived.Frame.Info.SamplesPerChannel * + e.FrameReceived.Frame.Info.NumChannels * + sizeof(short)); + + // Span over the incoming mono audio bytes + var monoByteSpan = new ReadOnlySpan( + (void*)e.FrameReceived.Frame.Info.DataPtr, + monoLengthBytes); + + // Treat them as 16-bit samples + var monoSamples = MemoryMarshal.Cast(monoByteSpan); + + // Allocate stereo buffer on the stack: 2 channels * samples * sizeof(short) + Span stereoSamples = stackalloc short[samplesPerChannel * 2]; + + for (int i = 0; i < samplesPerChannel; i++) + { + short sample = monoSamples[i]; // mono + int dstIndex = i * 2; + + stereoSamples[dstIndex] = sample; // Left + stereoSamples[dstIndex + 1] = sample; // Right + } + + // Cast the stereo short span to bytes and write to the buffer + ReadOnlySpan stereoBytes = MemoryMarshal.AsBytes(stereoSamples); + _buffer?.Write(stereoBytes); + } + else + { + //TODO add support here if they fix the above bug } } } + // Change - need to drop handle here because it would normally be done + // within AudioFrame, without memory will be leaked on the plugin side + NativeMethods.FfiDropHandle((IntPtr)e.FrameReceived.Frame.Handle.Id); } public void Dispose() diff --git a/Runtime/Scripts/Internal/FFIClient.cs b/Runtime/Scripts/Internal/FFIClient.cs index d037a3bb..ec450117 100644 --- a/Runtime/Scripts/Internal/FFIClient.cs +++ b/Runtime/Scripts/Internal/FFIClient.cs @@ -185,7 +185,7 @@ public void Release(FfiResponse response) ffiResponsePool.Release(response); } - public FfiResponse SendRequest(FfiRequest request) + public FfiResponse SendRequest(FfiRequest request, bool requiresResponse = true) { try { @@ -203,9 +203,18 @@ public FfiResponse SendRequest(FfiRequest request) out UIntPtr dataLen ); var dataSpan = new Span(dataPtr, (int)dataLen.ToUInt64()); - var response = responseParser.ParseFrom(dataSpan)!; - NativeMethods.FfiDropHandle(handle); - return response; + + if (requiresResponse) + { + var response = responseParser.ParseFrom(dataSpan)!; + NativeMethods.FfiDropHandle(handle); + return response; + } + else + { + NativeMethods.FfiDropHandle(handle); + return null; + } } } } diff --git a/Runtime/Scripts/Internal/FFIClients/IFFIClient.cs b/Runtime/Scripts/Internal/FFIClients/IFFIClient.cs index 4d0cdb38..2def9296 100644 --- a/Runtime/Scripts/Internal/FFIClients/IFFIClient.cs +++ b/Runtime/Scripts/Internal/FFIClients/IFFIClient.cs @@ -8,7 +8,7 @@ namespace LiveKit.Internal.FFIClients /// public interface IFFIClient : IDisposable { - FfiResponse SendRequest(FfiRequest request); + FfiResponse SendRequest(FfiRequest request, bool requireResponse = true); void Release(FfiResponse response); } diff --git a/Runtime/Scripts/Internal/FFIClients/Requests/FFIBridge.cs b/Runtime/Scripts/Internal/FFIClients/Requests/FFIBridge.cs index 2d3f1523..d9fba415 100644 --- a/Runtime/Scripts/Internal/FFIClients/Requests/FFIBridge.cs +++ b/Runtime/Scripts/Internal/FFIClients/Requests/FFIBridge.cs @@ -15,8 +15,8 @@ public class FFIBridge : IFFIBridge public static FFIBridge Instance => instance.Value; - private readonly IFFIClient ffiClient; - private readonly IMultiPool multiPool; + public readonly IFFIClient ffiClient; + public readonly IMultiPool multiPool; public FFIBridge(IFFIClient client, IMultiPool multiPool) { diff --git a/Runtime/Scripts/RtcAudioSource.cs b/Runtime/Scripts/RtcAudioSource.cs index ef512733..6296e089 100644 --- a/Runtime/Scripts/RtcAudioSource.cs +++ b/Runtime/Scripts/RtcAudioSource.cs @@ -126,26 +126,37 @@ static short FloatToS16(float v) for (int i = 0; i < data.Length; i++) _frameData[i] = FloatToS16(data[i]); + //Change - hand to make FFIBridge properties public to do this which isn't great // Capture the frame. - using var request = FFIBridge.Instance.NewRequest(); - using var audioFrameBufferInfo = request.TempResource(); + //using var request = FFIBridge.Instance.NewRequest(); + CaptureAudioFrameRequest pushFrame = FFIBridge.Instance.multiPool.Get(); + AudioFrameBufferInfo audioFrameBufferInfo = FFIBridge.Instance.multiPool.Get(); - var pushFrame = request.request; + FfiRequest ffiRequest = FFIBridge.Instance.multiPool.Get(); + + // using var audioFrameBufferInfo = request.TempResource(); + + // var pushFrame = request.request; pushFrame.SourceHandle = (ulong)Handle.DangerousGetHandle(); pushFrame.Buffer = audioFrameBufferInfo; unsafe { - pushFrame.Buffer.DataPtr = (ulong)NativeArrayUnsafeUtility - .GetUnsafePtr(_frameData); + pushFrame.Buffer.DataPtr = (ulong)NativeArrayUnsafeUtility + .GetUnsafePtr(_frameData); } pushFrame.Buffer.NumChannels = (uint)channels; pushFrame.Buffer.SampleRate = (uint)sampleRate; pushFrame.Buffer.SamplesPerChannel = (uint)data.Length / (uint)channels; - using var response = request.Send(); - FfiResponse res = response; + // using var response = request.Send(); + // FfiResponse res = response; + ffiRequest.CaptureAudioFrame = pushFrame; + FFIBridge.Instance.ffiClient.SendRequest(ffiRequest, false); - // Wait for async callback, log an error if the capture fails. + + // Changes - this was creating memory because of the callback being created, + // might be able to make a call back and cache the asyncID to get around that problem + /* var asyncId = res.CaptureAudioFrame.AsyncId; void Callback(CaptureAudioFrameCallback callback) { @@ -155,6 +166,17 @@ void Callback(CaptureAudioFrameCallback callback) FfiClient.Instance.CaptureAudioFrameReceived -= Callback; } FfiClient.Instance.CaptureAudioFrameReceived += Callback; + */ + + ffiRequest.CaptureAudioFrame = null; + pushFrame.Buffer.ClearDataPtr(); + pushFrame.Buffer = null; + ffiRequest.ClearMessage(); + + + FFIBridge.Instance.multiPool.Release(ffiRequest); + FFIBridge.Instance.multiPool.Release(pushFrame); + FFIBridge.Instance.multiPool.Release(audioFrameBufferInfo); } /// From 202df2842d69dcf34cff4615aba244c2f40e46f8 Mon Sep 17 00:00:00 2001 From: cloudwebrtc Date: Thu, 8 Jan 2026 14:12:28 +0800 Subject: [PATCH 2/3] cleanup. --- Runtime/Scripts/AudioStream.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Runtime/Scripts/AudioStream.cs b/Runtime/Scripts/AudioStream.cs index 4da2cbdf..7189a76e 100644 --- a/Runtime/Scripts/AudioStream.cs +++ b/Runtime/Scripts/AudioStream.cs @@ -138,14 +138,12 @@ private void OnAudioStreamEvent(AudioStreamEvent e) } // Cast the stereo short span to bytes and write to the buffer - ReadOnlySpan stereoBytes = MemoryMarshal.AsBytes(stereoSamples); - _buffer?.Write(stereoBytes); + _buffer?.Write(MemoryMarshal.AsBytes(stereoSamples)); } else { //TODO add support here if they fix the above bug } - } } // Change - need to drop handle here because it would normally be done @@ -163,8 +161,10 @@ private void Dispose(bool disposing) { if (!_disposed && disposing) { + FfiClient.Instance.AudioStreamEventReceived -= OnAudioStreamEvent; _audioSource.Stop(); UnityEngine.Object.Destroy(_audioSource.GetComponent()); + _buffer?.Dispose(); } _disposed = true; } From fa22aba53717b229d5f7a681301d82978853f437 Mon Sep 17 00:00:00 2001 From: cloudwebrtc Date: Thu, 8 Jan 2026 14:16:12 +0800 Subject: [PATCH 3/3] cleanup. --- Runtime/Scripts/AudioStream.cs | 2 -- Runtime/Scripts/RtcAudioSource.cs | 8 ++------ 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/Runtime/Scripts/AudioStream.cs b/Runtime/Scripts/AudioStream.cs index 7189a76e..941f1b20 100644 --- a/Runtime/Scripts/AudioStream.cs +++ b/Runtime/Scripts/AudioStream.cs @@ -96,8 +96,6 @@ private void OnAudioStreamEvent(AudioStreamEvent e) if (e.MessageCase != AudioStreamEvent.MessageOneofCase.FrameReceived) return; - //var frame = new AudioFrame(e.FrameReceived.Frame); - lock (_lock) { if (_numChannels == 0) diff --git a/Runtime/Scripts/RtcAudioSource.cs b/Runtime/Scripts/RtcAudioSource.cs index 6296e089..0d0de432 100644 --- a/Runtime/Scripts/RtcAudioSource.cs +++ b/Runtime/Scripts/RtcAudioSource.cs @@ -128,15 +128,11 @@ static short FloatToS16(float v) //Change - hand to make FFIBridge properties public to do this which isn't great // Capture the frame. - //using var request = FFIBridge.Instance.NewRequest(); CaptureAudioFrameRequest pushFrame = FFIBridge.Instance.multiPool.Get(); AudioFrameBufferInfo audioFrameBufferInfo = FFIBridge.Instance.multiPool.Get(); FfiRequest ffiRequest = FFIBridge.Instance.multiPool.Get(); - // using var audioFrameBufferInfo = request.TempResource(); - - // var pushFrame = request.request; pushFrame.SourceHandle = (ulong)Handle.DangerousGetHandle(); pushFrame.Buffer = audioFrameBufferInfo; unsafe @@ -148,14 +144,14 @@ static short FloatToS16(float v) pushFrame.Buffer.SampleRate = (uint)sampleRate; pushFrame.Buffer.SamplesPerChannel = (uint)data.Length / (uint)channels; - // using var response = request.Send(); - // FfiResponse res = response; + ffiRequest.CaptureAudioFrame = pushFrame; FFIBridge.Instance.ffiClient.SendRequest(ffiRequest, false); // Changes - this was creating memory because of the callback being created, // might be able to make a call back and cache the asyncID to get around that problem + // TODO: Investigate further and optimize if needed /* var asyncId = res.CaptureAudioFrame.AsyncId; void Callback(CaptureAudioFrameCallback callback)