From fea67869f2fa863e59790d79454d1c93dc397637 Mon Sep 17 00:00:00 2001 From: stary <834207172@qq.com> Date: Wed, 3 Jun 2026 22:05:03 +0800 Subject: [PATCH] =?UTF-8?q?=E6=9C=AC=E5=9C=B0=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Assets/Plugins/Android/AndroidManifest.xml | 3 + Assets/Scenes/SampleScene.unity | 159 +++++++- Assets/Script/GlobalConfigSystem.cs | 15 + Assets/Script/MainPanel/MainPanel.cs | 5 + .../MainPanel/MeetingInfoListController.cs | 3 +- .../Script/Recorder/EditorServerRecorder.cs | 260 ++++++++++++ .../Recorder/EditorServerRecorder.cs.meta | 3 + Assets/Script/Recorder/ServerMixedRecorder.cs | 381 ++++++++++++++++++ .../Recorder/ServerMixedRecorder.cs.meta | 3 + .../Script/Recorder/WebcamToRenderTexture.cs | 47 +++ .../Recorder/WebcamToRenderTexture.cs.meta | 3 + Assets/Script/Recorder/XrealMixedRecorder.cs | 28 +- Assets/Script/RenderStreamingSystem.cs | 11 + Assets/Script/WebRequestSystem.cs | 57 +-- Assets/Script/WebRtc/MessageChannel.cs | 27 ++ Assets/Script/WebRtc/MessageTypes.cs | 44 ++ Assets/Script/YUVToRenderTexture.cs | 9 +- .../Scripts/Signaling/WebSocketSignaling.cs | 3 +- 18 files changed, 996 insertions(+), 65 deletions(-) create mode 100644 Assets/Script/Recorder/EditorServerRecorder.cs create mode 100644 Assets/Script/Recorder/EditorServerRecorder.cs.meta create mode 100644 Assets/Script/Recorder/ServerMixedRecorder.cs create mode 100644 Assets/Script/Recorder/ServerMixedRecorder.cs.meta create mode 100644 Assets/Script/Recorder/WebcamToRenderTexture.cs create mode 100644 Assets/Script/Recorder/WebcamToRenderTexture.cs.meta diff --git a/Assets/Plugins/Android/AndroidManifest.xml b/Assets/Plugins/Android/AndroidManifest.xml index 0d9fdae..bfc8280 100644 --- a/Assets/Plugins/Android/AndroidManifest.xml +++ b/Assets/Plugins/Android/AndroidManifest.xml @@ -26,4 +26,7 @@ + + + diff --git a/Assets/Scenes/SampleScene.unity b/Assets/Scenes/SampleScene.unity index 97794a3..fbfcb6c 100644 --- a/Assets/Scenes/SampleScene.unity +++ b/Assets/Scenes/SampleScene.unity @@ -122,6 +122,134 @@ NavMeshSettings: debug: m_Flags: 0 m_NavMeshData: {fileID: 0} +--- !u!1 &314561960 +GameObject: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + serializedVersion: 6 + m_Component: + - component: {fileID: 314561961} + - component: {fileID: 314561962} + m_Layer: 0 + m_Name: microphoneAudioSource + m_TagString: Untagged + m_Icon: {fileID: 0} + m_NavMeshLayer: 0 + m_StaticEditorFlags: 0 + m_IsActive: 1 +--- !u!4 &314561961 +Transform: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 314561960} + serializedVersion: 2 + m_LocalRotation: {x: -0, y: -0, z: -0, w: 1} + m_LocalPosition: {x: 0, y: 0, z: 0} + m_LocalScale: {x: 1, y: 1, z: 1} + m_ConstrainProportionsScale: 0 + m_Children: [] + m_Father: {fileID: 1915034403} + m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0} +--- !u!82 &314561962 +AudioSource: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 314561960} + m_Enabled: 1 + serializedVersion: 4 + OutputAudioMixerGroup: {fileID: 0} + m_audioClip: {fileID: 0} + m_PlayOnAwake: 1 + m_Volume: 1 + m_Pitch: 1 + Loop: 0 + Mute: 0 + Spatialize: 0 + SpatializePostEffects: 0 + Priority: 128 + DopplerLevel: 1 + MinDistance: 1 + MaxDistance: 500 + Pan2D: 0 + rolloffMode: 0 + BypassEffects: 0 + BypassListenerEffects: 0 + BypassReverbZones: 0 + rolloffCustomCurve: + serializedVersion: 2 + m_Curve: + - serializedVersion: 3 + time: 0 + value: 1 + inSlope: 0 + outSlope: 0 + tangentMode: 0 + weightedMode: 0 + inWeight: 0.33333334 + outWeight: 0.33333334 + - serializedVersion: 3 + time: 1 + value: 0 + inSlope: 0 + outSlope: 0 + tangentMode: 0 + weightedMode: 0 + inWeight: 0.33333334 + outWeight: 0.33333334 + m_PreInfinity: 2 + m_PostInfinity: 2 + m_RotationOrder: 4 + panLevelCustomCurve: + serializedVersion: 2 + m_Curve: + - serializedVersion: 3 + time: 0 + value: 0 + inSlope: 0 + outSlope: 0 + tangentMode: 0 + weightedMode: 0 + inWeight: 0.33333334 + outWeight: 0.33333334 + m_PreInfinity: 2 + m_PostInfinity: 2 + m_RotationOrder: 4 + spreadCustomCurve: + serializedVersion: 2 + m_Curve: + - serializedVersion: 3 + time: 0 + value: 0 + inSlope: 0 + outSlope: 0 + tangentMode: 0 + weightedMode: 0 + inWeight: 0.33333334 + outWeight: 0.33333334 + m_PreInfinity: 2 + m_PostInfinity: 2 + m_RotationOrder: 4 + reverbZoneMixCustomCurve: + serializedVersion: 2 + m_Curve: + - serializedVersion: 3 + time: 0 + value: 1 + inSlope: 0 + outSlope: 0 + tangentMode: 0 + weightedMode: 0 + inWeight: 0.33333334 + outWeight: 0.33333334 + m_PreInfinity: 2 + m_PostInfinity: 2 + m_RotationOrder: 4 --- !u!224 &322971333 stripped RectTransform: m_CorrespondingSourceObject: {fileID: 2618240917987615803, guid: e0d7d0986111c73499b1c0e092bc092a, @@ -579,6 +707,12 @@ MonoBehaviour: m_MipBias: 0 m_VarianceClampScale: 0.9 m_ContrastAdaptiveSharpening: 0 +--- !u!81 &1595310059 stripped +AudioListener: + m_CorrespondingSourceObject: {fileID: 875232991587860349, guid: 779fa6087d61b4c4abc175a2d5d948f5, + type: 3} + m_PrefabInstance: {fileID: 1511841562} + m_PrefabAsset: {fileID: 0} --- !u!1 &1915034400 GameObject: m_ObjectHideFlags: 0 @@ -644,7 +778,8 @@ RectTransform: m_LocalPosition: {x: 0, y: 0, z: 0} m_LocalScale: {x: 1, y: 1, z: 1} m_ConstrainProportionsScale: 0 - m_Children: [] + m_Children: + - {fileID: 314561961} m_Father: {fileID: 0} m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0} m_AnchorMin: {x: 0.5, y: 0.5} @@ -664,20 +799,20 @@ MonoBehaviour: m_Script: {fileID: 11500000, guid: 8bcc82901c3f48a88d3408251afa3365, type: 3} m_Name: m_EditorClassIdentifier: - m_TextureSize: {x: 1920, y: 1200} + m_TextureSize: {x: 2560, y: 1440} m_Source: 3 m_Camera: {fileID: 0} - m_Texture: {fileID: 8400000, guid: dc38ccb1cb998bf4e8b83a3bbf7b46ad, type: 2} - m_WebCamDeviceIndex: 1 + m_Texture: {fileID: 0} + m_WebCamDeviceIndex: 0 m_Depth: 16 m_AntiAliasing: 1 m_Codec: - m_MimeType: video/H264 - m_SdpFmtpLine: implementation_name=NvCodec;level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=420033 - m_FrameRate: 30 + m_MimeType: video/VP9 + m_SdpFmtpLine: implementation_name=Internal;profile-id=0 + m_FrameRate: 60 m_Bitrate: - min: 0 - max: 1000 + min: 4000 + max: 8000 m_ScaleFactor: 1 m_AutoRequestUserAuthorization: 1 --- !u!114 &1915034405 @@ -731,7 +866,7 @@ MonoBehaviour: m_Name: m_EditorClassIdentifier: m_Source: 2 - m_AudioListener: {fileID: 0} + m_AudioListener: {fileID: 1595310059} m_AudioSource: {fileID: 0} m_MicrophoneDeviceIndex: 0 m_AutoRequestUserAuthorization: 1 @@ -741,8 +876,8 @@ MonoBehaviour: m_ChannelCount: 0 m_SampleRate: 0 m_Bitrate: - min: 8 - max: 208 + min: 247 + max: 1000 m_Loopback: 0 --- !u!1660057539 &9223372036854775807 SceneRoots: diff --git a/Assets/Script/GlobalConfigSystem.cs b/Assets/Script/GlobalConfigSystem.cs index a292ff4..90a8071 100644 --- a/Assets/Script/GlobalConfigSystem.cs +++ b/Assets/Script/GlobalConfigSystem.cs @@ -31,6 +31,9 @@ public interface IGlobalConfigSystem : ISystem public string GetUserId(); public void SetUserId(string userId); + + public string GetParticipantId(); + public void SetParticipantId(string participantId); } public class GlobalConfigSystem : AbstractSystem, IGlobalConfigSystem @@ -65,6 +68,8 @@ public class GlobalConfigSystem : AbstractSystem, IGlobalConfigSystem /// private int _connectionTimeType; + private string _participantId; + private CancellationTokenSource _cts; @@ -211,6 +216,16 @@ public class GlobalConfigSystem : AbstractSystem, IGlobalConfigSystem _connectionAvatar = connectionTexture; } + public string GetParticipantId() + { + return _participantId; + } + + public void SetParticipantId(string participantId) + { + _participantId = participantId; + } + protected override void OnInit() { } diff --git a/Assets/Script/MainPanel/MainPanel.cs b/Assets/Script/MainPanel/MainPanel.cs index 5bce6f9..1266aa0 100644 --- a/Assets/Script/MainPanel/MainPanel.cs +++ b/Assets/Script/MainPanel/MainPanel.cs @@ -128,6 +128,11 @@ namespace Script await WebRTCUtil.DownloadAndSetAvatar(obj[i].avatar, entry.transform.Find("image").GetComponent()); _userMap.TryAdd(obj[i], entry); + + if (obj[i].role == "host") + { + this.GetSystem().SetParticipantId(obj[i].participantId); + } } // 更新会议聊天面板人数 diff --git a/Assets/Script/MainPanel/MeetingInfoListController.cs b/Assets/Script/MainPanel/MeetingInfoListController.cs index 36b92da..dbe6ab7 100644 --- a/Assets/Script/MainPanel/MeetingInfoListController.cs +++ b/Assets/Script/MainPanel/MeetingInfoListController.cs @@ -1,6 +1,7 @@ using System.Collections.Generic; using Newtonsoft.Json; using RenderStreaming; +using Script.Recorder; using Script.Util; using Stary.Evo; using Stary.Evo.UIFarme; @@ -58,7 +59,7 @@ namespace Script _audioStreamSender = GameObject.Find("RenderStreaming").GetComponent(); _videoStreamSender = GameObject.Find("RenderStreaming").GetComponent(); #if UNITY_EDITOR - _recorder = new EditorGameViewRecorder(); + _recorder = new ServerMixedRecorder(); #elif UNITY_ANDROID _recorder = new XrealMixedRecorder(); #else diff --git a/Assets/Script/Recorder/EditorServerRecorder.cs b/Assets/Script/Recorder/EditorServerRecorder.cs new file mode 100644 index 0000000..ed2f007 --- /dev/null +++ b/Assets/Script/Recorder/EditorServerRecorder.cs @@ -0,0 +1,260 @@ +using System; +using System.Collections.Generic; +using Cysharp.Threading.Tasks; +using Newtonsoft.Json; +using RenderStreaming; +using Stary.Evo; +using Unity.RenderStreaming; +using Unity.WebRTC; +using UnityEngine; + +namespace Script.Recorder +{ + public class EditorServerRecorder : IVideoRecorder, IController + { + public bool IsRecording { get; } + public Action OnStartedRecordingVideo { get; set; } + public Action OnStoppedRecordingVideoAction { get; set; } + + private string recordingId; + + private readonly Dictionary recordingPeers = new(); + + public WebcamToRenderTexture sourceRenderTexture; + public string microphoneDeviceName = ""; // 留空使用默认麦克风 + public int microphoneSampleRate = 48000; + + private AudioSource microphoneAudioSource; + private bool microphoneStarted; + + private VideoStreamTrack videoTrack; + private AudioStreamTrack audioTrack; + + public async void StartRecording() + { + await CreateLocalTracks(); + + var messageChannel = GameObject.FindObjectOfType(); + messageChannel.OnRecordingPeerRequestReceived += OnRecordingPeerRequestReceived; + messageChannel.OnRecordingAnswerReceived += OnRecordingAnswerReceived; + messageChannel.OnRecordingCandidateReceived += OnRecordingCandidateReceived; + messageChannel.OnRecordingStoppedReceived += OnRecordingStoppedReceived; + + var data = new + { + connectionId = this.GetSystem().GetConnectionId(), + layout = "grid", + format = "webm", + }; + + var result = await WebRequestSystem.Post( + this.GetSystem().IP + "/api/recording-sessions", + JsonConvert.SerializeObject(data)); + if (result != null) + { + RecordingSession session = JsonConvert.DeserializeObject(result); + if (string.IsNullOrEmpty(session.success)) + { + Debug.LogError($"[ServerMixedRecorder] StartRecording 失败: {session}"); + return; + } + + recordingId = session.session.id; + } + + } + + + public async UniTask CreateLocalTracks() + { + if (sourceRenderTexture == null) + { + sourceRenderTexture =new GameObject("WebcamToRenderTexture").AddComponent(); + videoTrack = new VideoStreamTrack(sourceRenderTexture.renderTexture); + } + + microphoneAudioSource = + GameObject.Find("RenderStreaming/microphoneAudioSource").GetComponent(); + microphoneAudioSource.loop = true; + microphoneAudioSource.mute = true; + + string device = string.IsNullOrEmpty(microphoneDeviceName) + ? null + : microphoneDeviceName; + + microphoneAudioSource.clip = Microphone.Start( + device, + true, + 1, + microphoneSampleRate + ); + + microphoneStarted = true; + + await CreateMicrophoneTrackWhenReady(device); + } + + public async UniTask CreateMicrophoneTrackWhenReady(string device) + { + while (Microphone.GetPosition(device) <= 0) + await UniTask.NextFrame(); + + microphoneAudioSource.Play(); + audioTrack = new AudioStreamTrack(microphoneAudioSource); + } + + + public async void OnRecordingPeerRequestReceived(RecordingRequest request) + { + if (string.IsNullOrEmpty(request.recordingId)) + return; + + OnRecordingStoppedReceived(request.recordingId); + + var config = new RTCConfiguration + { + iceServers = new[] + { + new RTCIceServer { urls = new[] { "stun:stun.l.google.com:19302" } } + } + }; + + var pc = new RTCPeerConnection(ref config); + recordingPeers[request.recordingId] = pc; + + pc.OnIceCandidate = candidate => + { + if (candidate == null) return; + + Send("recording-candidate", new RecordingCandidate + { + recordingId = request.recordingId, + connectionId = request.connectionId, + participantId = this.GetSystem().GetParticipantId(), + candidate = candidate.Candidate, + sdpMid = candidate.SdpMid, + sdpMLineIndex = candidate.SdpMLineIndex ?? 0 + }); + }; + + if (videoTrack != null) + pc.AddTransceiver(videoTrack, + new RTCRtpTransceiverInit { direction = RTCRtpTransceiverDirection.SendOnly }); + + if (audioTrack != null) + pc.AddTransceiver(audioTrack, + new RTCRtpTransceiverInit { direction = RTCRtpTransceiverDirection.SendOnly }); + + var offerOp = pc.CreateOffer(); + await offerOp; + + var offer = offerOp.Desc; + var localOp = pc.SetLocalDescription(ref offer); + await localOp; + + Send("recording-offer", new RecordingOffer + { + recordingId = request.recordingId, + connectionId = request.connectionId, + participantId = this.GetSystem().GetParticipantId(), + sdp = offer.sdp + }); + } + + public async void OnRecordingAnswerReceived(RecordingAnswer answer) + { + if (!recordingPeers.TryGetValue(answer.recordingId, out var pc)) + return; + + var desc = new RTCSessionDescription + { + type = RTCSdpType.Answer, + sdp = answer.sdp + }; + + var op = pc.SetRemoteDescription(ref desc); + await op; + } + + public void OnRecordingCandidateReceived(RecordingCandidate data) + { + if (!recordingPeers.TryGetValue(data.recordingId, out var pc)) + return; + + var candidate = new RTCIceCandidate(new RTCIceCandidateInit + { + candidate = data.candidate, + sdpMid = data.sdpMid, + sdpMLineIndex = data.sdpMLineIndex + }); + + pc.AddIceCandidate(candidate); + } + + public void OnRecordingStoppedReceived(string recordingId) + { + if (!recordingPeers.TryGetValue(recordingId, out var pc)) + return; + + pc.Close(); + recordingPeers.Remove(recordingId); + } + + void Send(string type, object data) + { + var connectionId = this.GetSystem().GetConnectionId(); + + var json = JsonConvert.SerializeObject(new Dictionary + { + ["type"] = "on-message", + ["data"] = new Dictionary + { + ["connectionId"] = connectionId, + ["message"] = new Dictionary + { + ["type"] = type, + ["data"] = data + } + } + }); + + SignalingMessageHelper.SendMessage(json); + } + + public async void StopRecording() + { + await WebRequestSystem.Delete(this.GetSystem().IP, + $"/api/recording-sessions/{recordingId}"); + var messageChannel = GameObject.FindObjectOfType(); + messageChannel.OnRecordingPeerRequestReceived -= OnRecordingPeerRequestReceived; + messageChannel.OnRecordingAnswerReceived -= OnRecordingAnswerReceived; + messageChannel.OnRecordingCandidateReceived -= OnRecordingCandidateReceived; + messageChannel.OnRecordingStoppedReceived -= OnRecordingStoppedReceived; + } + + void OnDestroy() + { + foreach (var pc in recordingPeers.Values) + pc.Close(); + + recordingPeers.Clear(); + + videoTrack?.Dispose(); + audioTrack?.Dispose(); + + if (microphoneStarted) + { + string device = string.IsNullOrEmpty(microphoneDeviceName) + ? null + : microphoneDeviceName; + + Microphone.End(device); + } + } + + public IArchitecture GetArchitecture() + { + return MainArchitecture.Interface; + } + } +} \ No newline at end of file diff --git a/Assets/Script/Recorder/EditorServerRecorder.cs.meta b/Assets/Script/Recorder/EditorServerRecorder.cs.meta new file mode 100644 index 0000000..608266f --- /dev/null +++ b/Assets/Script/Recorder/EditorServerRecorder.cs.meta @@ -0,0 +1,3 @@ +fileFormatVersion: 2 +guid: 3583d180c5d94451bea6ead323d96021 +timeCreated: 1780402391 \ No newline at end of file diff --git a/Assets/Script/Recorder/ServerMixedRecorder.cs b/Assets/Script/Recorder/ServerMixedRecorder.cs new file mode 100644 index 0000000..b77ff8f --- /dev/null +++ b/Assets/Script/Recorder/ServerMixedRecorder.cs @@ -0,0 +1,381 @@ +using System; +using System.Collections.Generic; +using Cysharp.Threading.Tasks; +using Newtonsoft.Json; +using RenderStreaming; +using Stary.Evo; +using Unity.RenderStreaming; +using Unity.WebRTC; +using UnityEngine; + +namespace Script.Recorder +{ + public class ServerMixedRecorder : IVideoRecorder, IController + { + public bool IsRecording { get; } + public Action OnStartedRecordingVideo { get; set; } + public Action OnStoppedRecordingVideoAction { get; set; } + + private string recordingId; + + private readonly Dictionary recordingPeers = new(); + + + // [Header("Tracks")] public RenderTexture sourceRenderTexture; + // public string microphoneDeviceName = ""; // 留空使用默认麦克风 + // public int microphoneSampleRate = 48000; + // + // private AudioSource microphoneAudioSource; + // private bool microphoneStarted; + + private VideoStreamTrack videoTrack; + private AudioStreamTrack audioTrack; + + public async void StartRecording() + { + await CreateLocalTracks(); + + var messageChannel = GameObject.FindObjectOfType(); + messageChannel.OnRecordingPeerRequestReceived += OnRecordingPeerRequestReceived; + messageChannel.OnRecordingAnswerReceived += OnRecordingAnswerReceived; + messageChannel.OnRecordingCandidateReceived += OnRecordingCandidateReceived; + messageChannel.OnRecordingStoppedReceived += OnRecordingStoppedReceived; + + var data = new + { + connectionId = this.GetSystem().GetConnectionId(), + layout = "grid", + format = "webm", + }; + + var result = await WebRequestSystem.Post( + this.GetSystem().IP + "/api/recording-sessions", + JsonConvert.SerializeObject(data)); + if (result != null) + { + RecordingSession session = JsonConvert.DeserializeObject(result); + if (string.IsNullOrEmpty(session.success)) + { + Debug.LogError($"[ServerMixedRecorder] StartRecording 失败: {session}"); + return; + } + + recordingId = session.session.id; + } + + } + + + public async UniTask CreateLocalTracks() + { + var RenderStreaming =GameObject.Find("RenderStreaming"); + VideoStreamSender videoStreamSender = RenderStreaming.GetComponent(); + videoTrack = videoStreamSender.Track as VideoStreamTrack; + + + AudioStreamSender audioStreamSender = RenderStreaming.GetComponent(); + audioTrack = audioStreamSender.Track as AudioStreamTrack; + // sourceRenderTexture = this.GetSystem().GetRenderStreamingTexture(); + // if (sourceRenderTexture != null) + // { + // videoTrack = new VideoStreamTrack(sourceRenderTexture); + // } + // + // + // microphoneAudioSource = + // GameObject.Find("RenderStreaming/microphoneAudioSource").GetComponent(); + // microphoneAudioSource.loop = true; + // microphoneAudioSource.mute = true; + // + // string device = string.IsNullOrEmpty(microphoneDeviceName) + // ? null + // : microphoneDeviceName; + // + // microphoneAudioSource.clip = Microphone.Start( + // device, + // true, + // 1, + // microphoneSampleRate + // ); + // + // microphoneStarted = true; + + //await CreateMicrophoneTrackWhenReady(device); + } + + // public async UniTask CreateMicrophoneTrackWhenReady(string device) + // { + // while (Microphone.GetPosition(device) <= 0) + // await UniTask.NextFrame(); + // + // microphoneAudioSource.Play(); + // audioTrack = new AudioStreamTrack(microphoneAudioSource); + // } + + + public async void OnRecordingPeerRequestReceived(RecordingRequest request) + { + if (string.IsNullOrEmpty(request.recordingId)) + return; + + OnRecordingStoppedReceived(request.recordingId); + + var config = new RTCConfiguration + { + iceServers = new[] + { + new RTCIceServer { urls = new[] { "stun:stun.l.google.com:19302" } } + } + }; + + var pc = new RTCPeerConnection(ref config); + recordingPeers[request.recordingId] = pc; + + pc.OnIceCandidate = candidate => + { + if (candidate == null) return; + + Send("recording-candidate", new RecordingCandidate + { + recordingId = request.recordingId, + connectionId = request.connectionId, + participantId = this.GetSystem().GetParticipantId(), + candidate = candidate.Candidate, + sdpMid = candidate.SdpMid, + sdpMLineIndex = candidate.SdpMLineIndex ?? 0 + }); + }; + + if (videoTrack != null) + pc.AddTransceiver(videoTrack, + new RTCRtpTransceiverInit { direction = RTCRtpTransceiverDirection.SendOnly }); + + if (audioTrack != null) + pc.AddTransceiver(audioTrack, + new RTCRtpTransceiverInit { direction = RTCRtpTransceiverDirection.SendOnly }); + + var offerOp = pc.CreateOffer(); + await offerOp; + + var offer = offerOp.Desc; + var localOp = pc.SetLocalDescription(ref offer); + await localOp; + + Send("recording-offer", new RecordingOffer + { + recordingId = request.recordingId, + connectionId = request.connectionId, + participantId = this.GetSystem().GetParticipantId(), + sdp = offer.sdp + }); + } + + public async void OnRecordingAnswerReceived(RecordingAnswer answer) + { + if (!recordingPeers.TryGetValue(answer.recordingId, out var pc)) + return; + + var desc = new RTCSessionDescription + { + type = RTCSdpType.Answer, + sdp = answer.sdp + }; + + var op = pc.SetRemoteDescription(ref desc); + await op; + } + + public void OnRecordingCandidateReceived(RecordingCandidate data) + { + if (!recordingPeers.TryGetValue(data.recordingId, out var pc)) + return; + + var candidate = new RTCIceCandidate(new RTCIceCandidateInit + { + candidate = data.candidate, + sdpMid = data.sdpMid, + sdpMLineIndex = data.sdpMLineIndex + }); + + pc.AddIceCandidate(candidate); + } + + public void OnRecordingStoppedReceived(string recordingId) + { + if (!recordingPeers.TryGetValue(recordingId, out var pc)) + return; + + pc.Close(); + recordingPeers.Remove(recordingId); + } + + void Send(string type, object data) + { + var connectionId = this.GetSystem().GetConnectionId(); + + var json = JsonConvert.SerializeObject(new Dictionary + { + ["type"] = "on-message", + ["data"] = new Dictionary + { + ["connectionId"] = connectionId, + ["message"] = new Dictionary + { + ["type"] = type, + ["data"] = data + } + } + }); + + SignalingMessageHelper.SendMessage(json); + } + + public async void StopRecording() + { + await WebRequestSystem.Delete(this.GetSystem().IP, + $"/api/recording-sessions/{recordingId}"); + var messageChannel = GameObject.FindObjectOfType(); + messageChannel.OnRecordingPeerRequestReceived -= OnRecordingPeerRequestReceived; + messageChannel.OnRecordingAnswerReceived -= OnRecordingAnswerReceived; + messageChannel.OnRecordingCandidateReceived -= OnRecordingCandidateReceived; + messageChannel.OnRecordingStoppedReceived -= OnRecordingStoppedReceived; + } + + void OnDestroy() + { + foreach (var pc in recordingPeers.Values) + pc.Close(); + + recordingPeers.Clear(); + + videoTrack?.Dispose(); + audioTrack?.Dispose(); + + // if (microphoneStarted) + // { + // string device = string.IsNullOrEmpty(microphoneDeviceName) + // ? null + // : microphoneDeviceName; + // + // Microphone.End(device); + // } + } + + public IArchitecture GetArchitecture() + { + return MainArchitecture.Interface; + } + } + + [Serializable] + public class Session + { + /// + /// + /// + public string id { get; set; } + + /// + /// + /// + public string connectionId { get; set; } + + /// + /// + /// + public string status { get; set; } + + /// + /// + /// + public string layout { get; set; } + + /// + /// + /// + public string format { get; set; } + + /// + /// + /// + public string createdAt { get; set; } + + /// + /// + /// + public string startedAt { get; set; } + + /// + /// + /// + public string updatedAt { get; set; } + } + + [Serializable] + public class Agent + { + /// + /// + /// + public string id { get; set; } + + /// + /// + /// + public string recordingId { get; set; } + + /// + /// + /// + public string connectionId { get; set; } + + /// + /// + /// + public string status { get; set; } + + /// + /// + /// + public string mediaMode { get; set; } + + /// + /// + /// + public string createdAt { get; set; } + + /// + /// + /// + public string updatedAt { get; set; } + } + + [Serializable] + public class RecordingSession + { + /// + /// + /// + public string success { get; set; } + + /// + /// + /// + public Session session { get; set; } + + /// + /// + /// + public Agent agent { get; set; } + + /// + /// + /// + public string notified { get; set; } + + /// + /// + /// + public string peerRequestNotified { get; set; } + } +} \ No newline at end of file diff --git a/Assets/Script/Recorder/ServerMixedRecorder.cs.meta b/Assets/Script/Recorder/ServerMixedRecorder.cs.meta new file mode 100644 index 0000000..5427e91 --- /dev/null +++ b/Assets/Script/Recorder/ServerMixedRecorder.cs.meta @@ -0,0 +1,3 @@ +fileFormatVersion: 2 +guid: d9d49eabfbbd4b4d8b4b88f082441596 +timeCreated: 1780371636 \ No newline at end of file diff --git a/Assets/Script/Recorder/WebcamToRenderTexture.cs b/Assets/Script/Recorder/WebcamToRenderTexture.cs new file mode 100644 index 0000000..2105ec5 --- /dev/null +++ b/Assets/Script/Recorder/WebcamToRenderTexture.cs @@ -0,0 +1,47 @@ +using Unity.WebRTC; +using UnityEngine; + +public class WebcamToRenderTexture : MonoBehaviour +{ + public RenderTexture renderTexture; + + private WebCamTexture webcamTexture; + + void Awake() + { + if (renderTexture == null) + { + RenderTextureFormat supportedFormat = WebRTC.GetSupportedRenderTextureFormat(SystemInfo.graphicsDeviceType); + + // 创建新的RenderTexture + renderTexture = new RenderTexture(1920, 1200, 0, supportedFormat); + renderTexture.enableRandomWrite = true; + renderTexture.Create(); + //_textureImage.texture = localRenderTexture; + } + + webcamTexture = new WebCamTexture(); + webcamTexture.Play(); + } + + void Update() + { + if (webcamTexture != null && webcamTexture.didUpdateThisFrame) + { + Graphics.Blit(webcamTexture, renderTexture); + } + } + + void OnDestroy() + { + if (webcamTexture != null) + { + webcamTexture.Stop(); + } + + if (renderTexture != null) + { + renderTexture.Release(); + } + } +} \ No newline at end of file diff --git a/Assets/Script/Recorder/WebcamToRenderTexture.cs.meta b/Assets/Script/Recorder/WebcamToRenderTexture.cs.meta new file mode 100644 index 0000000..d4571a2 --- /dev/null +++ b/Assets/Script/Recorder/WebcamToRenderTexture.cs.meta @@ -0,0 +1,3 @@ +fileFormatVersion: 2 +guid: 63353e4a29ff45c9803dae86c9590d72 +timeCreated: 1780403919 \ No newline at end of file diff --git a/Assets/Script/Recorder/XrealMixedRecorder.cs b/Assets/Script/Recorder/XrealMixedRecorder.cs index 08ecadc..a0a572f 100644 --- a/Assets/Script/Recorder/XrealMixedRecorder.cs +++ b/Assets/Script/Recorder/XrealMixedRecorder.cs @@ -14,7 +14,7 @@ using CameraType = Unity.XR.XREAL.CameraType; using GalleryDataProvider = Unity.XR.XREAL.MockGalleryDataProvider; #endif -public class XrealMixedRecorder : IVideoRecorder, IController +public class XrealMixedRecorder : IVideoRecorder, IController , IDisposable { public enum ResolutionLevel { @@ -25,7 +25,7 @@ public class XrealMixedRecorder : IVideoRecorder, IController public ResolutionLevel resolutionLevel = ResolutionLevel.High; public BlendMode blendMode = BlendMode.Blend; - public AudioState audioState = AudioState.MicAudio; + public AudioState audioState = AudioState.ApplicationAndMicAudio; public CaptureSide captureside = CaptureSide.Single; public bool useGreenBackGround = false; @@ -82,7 +82,7 @@ public class XrealMixedRecorder : IVideoRecorder, IController } Debug.Log("Stop Video Capture!"); - _videoCapture.StopRecordingAsync(OnStoppedRecordingVideo); + _videoCapture.StopRecordingAsync(OnStoppedVideoCaptureMode); } /// Executes the 'stopped recording video' action. @@ -103,6 +103,11 @@ public class XrealMixedRecorder : IVideoRecorder, IController /// The result. private async void OnStoppedVideoCaptureMode(XREALVideoCapture.VideoCaptureResult result) { + if (!result.success) + { + Debug.Log("Stopped Recording Video Faild!"); + return; + } Debug.Log("Stopped Video Capture Mode!"); var encoder = _videoCapture.GetContext().GetEncoder() as VideoEncoder; @@ -114,8 +119,6 @@ public class XrealMixedRecorder : IVideoRecorder, IController await DelayInsertVideoToGallery(path, filename, "Record"); OnStoppedRecordingVideoAction?.Invoke(path); // Release video capture resource. - _videoCapture.Dispose(); - _videoCapture = null; } /// 延迟将视频插入相册，确保视频文件已完全写入 @@ -235,4 +238,19 @@ public class XrealMixedRecorder : IVideoRecorder, IController { return MainArchitecture.Interface; } + + public void Dispose() + { + _videoCapture.StopVideoModeAsync((result) => + { + if (!result.success) + { + Debug.Log("Stopped Video Capture Mode faild!"); + return; + } + _videoCapture?.Dispose(); + _videoCapture = null; + }); + + } } \ No newline at end of file diff --git a/Assets/Script/RenderStreamingSystem.cs b/Assets/Script/RenderStreamingSystem.cs index 656124a..8ec9735 100644 --- a/Assets/Script/RenderStreamingSystem.cs +++ b/Assets/Script/RenderStreamingSystem.cs @@ -11,6 +11,8 @@ namespace Script { void SetUp(); void HangUp(); + + RenderTexture GetRenderStreamingTexture(); } public class RenderStreamingSystem : AbstractSystem, IRenderStreamingSystem @@ -232,6 +234,15 @@ namespace Script Debug.Log($"[MultiParticipantHost] Participant UI removed: {participantId}"); } + public RenderTexture GetRenderStreamingTexture() + { + if (_yuvToRenderTexture.localRenderTexture == null) + { + Debug.LogError("RenderTexture 未初始化"); + return null; + } + return _yuvToRenderTexture.localRenderTexture; + } #endregion diff --git a/Assets/Script/WebRequestSystem.cs b/Assets/Script/WebRequestSystem.cs index 5274cdb..038d3ad 100644 --- a/Assets/Script/WebRequestSystem.cs +++ b/Assets/Script/WebRequestSystem.cs @@ -348,16 +348,13 @@ namespace Stary.Evo /// 获取Token值的服务URL地址（很重要） /// 传入请求的参数，此处参数为JOSN格式 /// - public static async Task Post(string url, string postData) + public static async Task Post(string url, string postData) { try { + await GetCertificateData(); using var webRequest = new UnityWebRequest(url, UnityWebRequest.kHttpVerbPOST); -// #if UNITY_2021_3_OR_NEWER -// using (UnityWebRequest webRequest = UnityWebRequest.PostWwwForm(url, postData)) //第二种写法此行注释 -// #else -// using (UnityWebRequest webRequest = UnityWebRequest.PostWwwForm(url, postData)) //第二种写法此行注释 -// #endif + webRequest.downloadHandler = new DownloadHandlerBuffer(); var postBytes = Encoding.UTF8.GetBytes(postData); webRequest.uploadHandler = new UploadHandlerRaw(postBytes); @@ -367,6 +364,7 @@ namespace Stary.Evo webRequest.disposeDownloadHandlerOnDispose = true; webRequest.disposeCertificateHandlerOnDispose = true; webRequest.timeout = 30; + webRequest.certificateHandler = new SelfSignedCertHandler(certificateData); await webRequest.SendWebRequest(); webRequest.uploadHandler?.Dispose(); // 更新错误检查方式 @@ -374,28 +372,16 @@ namespace Stary.Evo webRequest.result == UnityWebRequest.Result.ProtocolError) { Debug.LogError(webRequest.error); - return new ResultMessageEntity - { - code = 5001, - message = webRequest.error - }; + return webRequest.error; } - var resultMessageEntity = - JsonConvert.DeserializeObject(webRequest.downloadHandler.text); - if (resultMessageEntity.code != 200) Debug.LogError(resultMessageEntity.message); - - return resultMessageEntity; + return webRequest.downloadHandler.text; } catch (Exception e) { Debug.LogError("UnityEvo:WebRequestSystem.Post" + e.Message); - return new ResultMessageEntity - { - code = 5001, - message = e.Message - }; + return e.Message; } } @@ -408,10 +394,11 @@ namespace Stary.Evo /// 请求数据的URL地址 /// 请求数据的路径 /// - public static async Task Delete(string url, string path) + public static async Task Delete(string url, string path) { try { + await GetCertificateData(); // 修复URL拼接 var fullUrl = url.TrimEnd('/') + "/" + path.TrimStart('/'); using var webRequest = new UnityWebRequest(fullUrl, UnityWebRequest.kHttpVerbDELETE); @@ -420,6 +407,7 @@ namespace Stary.Evo webRequest.SetRequestHeader("Authorization", authorization); // 修正请求头名称规范 webRequest.timeout = 20; + webRequest.certificateHandler = new SelfSignedCertHandler(certificateData); await webRequest.SendWebRequest(); // 增强错误处理 @@ -431,36 +419,20 @@ namespace Stary.Evo $"Response: {webRequest.downloadHandler.text}"; Debug.LogError(errorMsg); - return new ResultMessageEntity - { - code = 5001, - message = errorMsg - }; + return errorMsg; } // 修复空响应处理 var responseText = webRequest.downloadHandler.text; - if (string.IsNullOrEmpty(responseText)) - return new ResultMessageEntity - { - code = 200, - message = "删除成功" - }; - var resultMessageEntity = - JsonConvert.DeserializeObject(webRequest.downloadHandler.text); - if (resultMessageEntity.code != 200) Debug.LogError(resultMessageEntity.message); - return resultMessageEntity; + + return responseText; } catch (Exception e) { Debug.LogError("UnityEvo:WebRequestSystem.Get" + e.Message); - return new ResultMessageEntity - { - code = 5001, - message = e.Message - }; + return e.Message; } } @@ -568,6 +540,5 @@ namespace Stary.Evo else Debug.Log($"上传成功: {request.downloadHandler.text}"); } - } } \ No newline at end of file diff --git a/Assets/Script/WebRtc/MessageChannel.cs b/Assets/Script/WebRtc/MessageChannel.cs index f469058..20c1daa 100644 --- a/Assets/Script/WebRtc/MessageChannel.cs +++ b/Assets/Script/WebRtc/MessageChannel.cs @@ -32,6 +32,12 @@ namespace Unity.RenderStreaming public event Action OnChatMessageReceived; + public event Action OnRecordingPeerRequestReceived; + public event Action OnRecordingAnswerReceived; + public event Action OnRecordingCandidateReceived; + public event Action OnRecordingStoppedReceived; + + public override void OnMessage(string message) { try @@ -59,6 +65,27 @@ namespace Unity.RenderStreaming var mediaState = json.ToObject(); OnMediaStateChangeReceived?.Invoke(ConnectionId, mediaState); break; + + case MessageTypes.RecordingPeerRequest: + json = record.data as JObject; + var recordingPeerRequest = json.ToObject(); + OnRecordingPeerRequestReceived?.Invoke(recordingPeerRequest); + break; + case MessageTypes.RecordingAnswer: + json = record.data as JObject; + var recordingAnswer = json.ToObject(); + OnRecordingAnswerReceived?.Invoke(recordingAnswer); + break; + case MessageTypes.RecordingCandidate: + json = record.data as JObject; + var recordingCandidate = json.ToObject(); + OnRecordingCandidateReceived?.Invoke(recordingCandidate); + break; + case MessageTypes.RecordingStopped: + json = record.data as JObject; + var recordingStopped = json.ToObject(); + OnRecordingStoppedReceived?.Invoke(recordingStopped.recordingId); + break; } messageHistory.Add(record); diff --git a/Assets/Script/WebRtc/MessageTypes.cs b/Assets/Script/WebRtc/MessageTypes.cs index 99eca1a..c747d8b 100644 --- a/Assets/Script/WebRtc/MessageTypes.cs +++ b/Assets/Script/WebRtc/MessageTypes.cs @@ -13,6 +13,11 @@ namespace Unity.RenderStreaming public const string UserInfo = "user-info"; public const string MediaStateChange = "media-state-changed"; public const string ParticipantsSync = "participants-sync"; + + public const string RecordingPeerRequest = "recording-peer-request"; + public const string RecordingAnswer = "recording-answer"; + public const string RecordingCandidate = "recording-candidate"; + public const string RecordingStopped = "recording-stopped"; } [Serializable] @@ -54,4 +59,43 @@ namespace Unity.RenderStreaming public int width; public int height; } + + + [Serializable]public class RecordingRequest + { + public string recordingId; + public string connectionId; + public string mediaMode; + } + + [Serializable]public class RecordingOffer + { + public string recordingId; + public string connectionId; + public string participantId; + public string sdp; + } + + [Serializable]public class RecordingAnswer + { + public string recordingId; + public string connectionId; + public string participantId; + public string sdp; + } + + [Serializable]public class RecordingCandidate + { + public string recordingId; + public string connectionId; + public string participantId; + public string candidate; + public string sdpMid; + public int sdpMLineIndex; + } + + [Serializable]public class RecordingStopped + { + public string recordingId; + } } \ No newline at end of file diff --git a/Assets/Script/YUVToRenderTexture.cs b/Assets/Script/YUVToRenderTexture.cs index 8969fac..450f648 100644 --- a/Assets/Script/YUVToRenderTexture.cs +++ b/Assets/Script/YUVToRenderTexture.cs @@ -1,6 +1,7 @@ using System; using RenderStreaming; using Stary.Evo; +using Unity.RenderStreaming; using Unity.XR.XREAL; using UnityEngine; using UnityEngine.UI; @@ -19,12 +20,14 @@ namespace Script //private RawImage _localVideoImage; //private RawImage _textureImage; + private VideoStreamSender videoStreamSender; private void Start() { //_localVideoImage= GameObject.Find("CanvasMain/RawImage").GetComponent(); //_textureImage= GameObject.Find("CanvasMain/RawImage1").GetComponent(); localVideoMaterial = Resources.Load("LocalRenderMaterial"); m_RGBCameraTexture = XREALRGBCameraTexture.CreateSingleton(); + videoStreamSender=this.GetComponent(); Play(); } @@ -52,8 +55,8 @@ namespace Script return; // 获取Y纹理的尺寸作为目标RenderTexture的尺寸 - int width = yuvTextures[0].width; - int height = yuvTextures[0].height; + var width = videoStreamSender.width; + var height = videoStreamSender.height; // if (_localVideoImage==null || _localVideoImage.rectTransform.sizeDelta.x != width || // _localVideoImage.rectTransform.sizeDelta.y != height) // { @@ -68,7 +71,7 @@ namespace Script localRenderTexture.Release(); // 创建新的RenderTexture - localRenderTexture = new RenderTexture(width, height, 0, RenderTextureFormat.ARGB32); + localRenderTexture = new RenderTexture((int)width, (int)height, 0, RenderTextureFormat.ARGB32); localRenderTexture.enableRandomWrite = true; localRenderTexture.Create(); //_textureImage.texture = localRenderTexture; diff --git a/Packages/com.unity.renderstreaming@3.1.0-exp.9/Runtime/Scripts/Signaling/WebSocketSignaling.cs b/Packages/com.unity.renderstreaming@3.1.0-exp.9/Runtime/Scripts/Signaling/WebSocketSignaling.cs index e49a62b..7a91b6d 100644 --- a/Packages/com.unity.renderstreaming@3.1.0-exp.9/Runtime/Scripts/Signaling/WebSocketSignaling.cs +++ b/Packages/com.unity.renderstreaming@3.1.0-exp.9/Runtime/Scripts/Signaling/WebSocketSignaling.cs @@ -4,6 +4,7 @@ using System.Linq; using System.Security.Authentication; using System.Text; using System.Threading; +using Newtonsoft.Json; using Unity.WebRTC; using UnityEngine; using WebSocketSharp; @@ -439,7 +440,7 @@ namespace Unity.RenderStreaming.Signaling } else if (routedMessage.type == "on-message") { - var message = JsonUtility.FromJson(content); + var message = JsonConvert.DeserializeObject(content); var messageData = new OnMessageData { connectionId = routedMessage.from,