From b0b5ef69eaa28d9887ecab1c10b62aa8056326fc Mon Sep 17 00:00:00 2001 From: "SmallChi(Koike)" <564952747@qq.com> Date: Sun, 5 Jun 2022 22:20:04 +0800 Subject: [PATCH] =?UTF-8?q?1.=E8=B0=83=E6=95=B4fmp4=E7=BC=96=E7=A0=81=202.?= =?UTF-8?q?=E8=B0=83=E6=95=B4=E5=9C=A8=E8=A7=A3=E6=9E=90nalu=E7=9A=84?= =?UTF-8?q?=E5=90=8C=E6=97=B6=E8=A7=A3=E6=9E=90=E5=87=BAsps=E5=92=8Cpps?= =?UTF-8?q?=E9=81=BF=E5=85=8D=E4=B9=8B=E5=90=8E=E5=86=8D=E4=B8=80=E6=AC=A1?= =?UTF-8?q?=E6=9F=A5=E8=AF=A2nalu=E5=9D=97?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 4 +- src/Info.props | 2 +- .../JT1078.AV.Benchmark.csproj | 2 +- src/JT1078.FMp4.Test/JT1078.FMp4.Test.csproj | 6 +- src/JT1078.FMp4.Test/JT1078ToFMp4Box_Test.cs | 62 +++++------ src/JT1078.FMp4/FMp4Encoder.cs | 100 +++++++---------- src/JT1078.FMp4/JT1078.FMp4.xml | 27 +---- src/JT1078.Flv.Test/JT1078.Flv.Test.csproj | 6 +- src/JT1078.Flv/FlvEncoder.cs | 104 ++++++++++++++++++ src/JT1078.Flv/JT1078.Flv.xml | 16 +++ src/JT1078.Hls.Test/JT1078.Hls.Test.csproj | 8 +- .../JT1078.Protocol.Benchmark.csproj | 2 +- .../JT1078.Protocol.Test.csproj | 6 +- src/JT1078.Protocol/H264/H264Decoder.cs | 96 +++++++++++++++- src/JT1078.Protocol/H264/H264NALU.cs | 9 +- src/JT1078.Protocol/IJT1078AVKey.cs | 13 +++ src/JT1078.Protocol/JT1078.Protocol.xml | 21 +++- src/JT1078.Protocol/JT1078AVFrame.cs | 39 +++++++ src/JT1078.Protocol/JT1078Package.cs | 10 +- .../Services/ToWebSocketService.cs | 59 ++++------ 20 files changed, 413 insertions(+), 179 deletions(-) create mode 100644 src/JT1078.Protocol/IJT1078AVKey.cs create mode 100644 src/JT1078.Protocol/JT1078AVFrame.cs diff --git a/README.md b/README.md index 17fb744..77b503d 100644 --- a/README.md +++ b/README.md @@ -89,8 +89,8 @@ var hex = JT1078Serializer.Serialize(jT1078Package).ToHexString(); 2.拆解： 30 31 63 64 --帧头表示 -81 --‭Label1 =>10000001‬ V P X CC -E2 --Label2 =>‭11100010‬ M PT +81 --Label1 =>10000001 V P X CC +E2 --Label2 =>11100010 M PT 10 88 --SN 包序号 01 12 34 56 78 10 --SIM卡号 01 --逻辑通道号 diff --git a/src/Info.props b/src/Info.props index 4786cd8..05e36fb 100644 --- a/src/Info.props +++ b/src/Info.props @@ -8,7 +8,7 @@ https://github.com/SmallChi/JT1078 https://github.com/SmallChi/JT1078/blob/master/LICENSE https://github.com/SmallChi/JT1078/blob/master/LICENSE - 1.2.0-preview2 + 1.2.0-preview6 LICENSE true latest diff --git a/src/JT1078.AV.Benchmark/JT1078.AV.Benchmark.csproj b/src/JT1078.AV.Benchmark/JT1078.AV.Benchmark.csproj index 4514a7f..55a8728 100644 --- a/src/JT1078.AV.Benchmark/JT1078.AV.Benchmark.csproj +++ b/src/JT1078.AV.Benchmark/JT1078.AV.Benchmark.csproj @@ -8,7 +8,7 @@ - + diff --git a/src/JT1078.FMp4.Test/JT1078.FMp4.Test.csproj b/src/JT1078.FMp4.Test/JT1078.FMp4.Test.csproj index 0ec67af..f7236e1 100644 --- a/src/JT1078.FMp4.Test/JT1078.FMp4.Test.csproj +++ b/src/JT1078.FMp4.Test/JT1078.FMp4.Test.csproj @@ -7,13 +7,13 @@ - + - + all runtime; build; native; contentfiles; analyzers; buildtransitive - + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/src/JT1078.FMp4.Test/JT1078ToFMp4Box_Test.cs b/src/JT1078.FMp4.Test/JT1078ToFMp4Box_Test.cs index d3b211d..36b25d0 100644 --- a/src/JT1078.FMp4.Test/JT1078ToFMp4Box_Test.cs +++ b/src/JT1078.FMp4.Test/JT1078ToFMp4Box_Test.cs @@ -12,6 +12,7 @@ using System.Buffers.Binary; using System.Collections.Generic; using System.IO; using System.Linq; +using System.Net.Sockets; using System.Text; using System.Threading; using System.Threading.Tasks; @@ -498,12 +499,15 @@ namespace JT1078.FMp4.Test } [Fact] - public void Test4_3() + public void Test4_4() { + uint a = uint.MaxValue; + var b = a + 1; + FMp4Encoder fMp4Encoder = new FMp4Encoder(); H264Decoder h264Decoder = new H264Decoder(); var packages = ParseNALUTests1(); - var filepath = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "H264", "JT1078_7_4_3.mp4"); + var filepath = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "H264", "JT1078_7_4_4.mp4"); if (File.Exists(filepath)) { File.Delete(filepath); @@ -520,9 +524,8 @@ namespace JT1078.FMp4.Test iNalus.FirstOrDefault(f => f.NALUHeader.NalUnitType == NalUnitType.SPS), iNalus.FirstOrDefault(f => f.NALUHeader.NalUnitType == NalUnitType.PPS)); fileStream.Write(moov); - List tmp = new List(); - List stream = new List(); + Queue mp4Frames = new Queue(); List filter = new List(); filter.Add(NalUnitType.SEI); filter.Add(NalUnitType.PPS); @@ -531,53 +534,42 @@ namespace JT1078.FMp4.Test foreach (var package in packages) { List h264NALUs = h264Decoder.ParseNALU(package); - if (h264NALUs!=null && h264NALUs.Count>0) + if (h264NALUs != null && h264NALUs.Count > 0) { - stream.AddRange(h264NALUs.Where(w=> !filter.Contains(w.NALUHeader.NalUnitType))); + Mp4Frame mp4Frame = new Mp4Frame + { + Key = package.GetKey(), + KeyFrame = package.Label3.DataType == JT1078DataType.视频I帧 + }; + mp4Frame.NALUs = h264NALUs; + mp4Frames.Enqueue(mp4Frame); } } - List tmp1 = new List(); - H264NALU prevNalu = null; - foreach (var item in stream) + while (mp4Frames.TryDequeue(out Mp4Frame frame)) { - if (item.NALUHeader.KeyFrame) - { - if (tmp1.Count>0) - { - fileStream.Write(fMp4Encoder.OtherVideoBox(tmp1)); - tmp1.Clear(); - } - tmp1.Add(item); - fileStream.Write(fMp4Encoder.OtherVideoBox(tmp1)); - tmp1.Clear(); - prevNalu=item; - continue; - } - if (prevNalu!=null) //第一帧I帧 - { - if (tmp1.Count>1) - { - fileStream.Write(fMp4Encoder.StypBox()); - fileStream.Write(fMp4Encoder.OtherVideoBox(tmp1)); - tmp1.Clear(); - } - tmp1.Add(item); - } + fileStream.Write(fMp4Encoder.OtherVideoBox(frame.NALUs, frame.Key, frame.KeyFrame)); } fileStream.Close(); - } + } + + class Mp4Frame + { + public string Key { get; set; } + public bool KeyFrame { get; set; } + public List NALUs { get; set; } + } [Fact] public void WebSocketMp4() { - var filepath = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "H264", "JT1078_8.mp4"); + var filepath = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "H264", "JT1078_10.mp4"); if (File.Exists(filepath)) { File.Delete(filepath); } using var fileStream = new FileStream(filepath, FileMode.OpenOrCreate, FileAccess.Write); System.Net.WebSockets.ClientWebSocket clientWebSocket = new System.Net.WebSockets.ClientWebSocket(); - clientWebSocket.ConnectAsync(new Uri("ws://127.0.0.1:81/live/JT1078_7.live.mp4"), CancellationToken.None).GetAwaiter().GetResult(); + clientWebSocket.ConnectAsync(new Uri("ws://127.0.0.1:8080/live/JT1078_7.live.mp4"), CancellationToken.None).GetAwaiter().GetResult(); Task.Run(async() => { while (true) diff --git a/src/JT1078.FMp4/FMp4Encoder.cs b/src/JT1078.FMp4/FMp4Encoder.cs index 78f55f4..832c348 100644 --- a/src/JT1078.FMp4/FMp4Encoder.cs +++ b/src/JT1078.FMp4/FMp4Encoder.cs @@ -16,26 +16,6 @@ namespace JT1078.FMp4 /// FMp4编码 /// fmp4 /// stream data - /// ftyp - /// moov - /// mvex - /// trex Video - /// trex Audio - /// styp 1 - /// sidx 1 Video - /// sidx 1 Audio - /// moof 1 - /// traf 1 Video - /// traf 1 Audio - /// mdat 1 - /// ... - /// styp n - /// sidx 1 Video - /// sidx n Audio - /// moof n - /// traf n Video - /// traf n Audio - /// mdat n /// ref: https://www.w3.org/TR/mse-byte-stream-format-isobmff/#movie-fragment-relative-addressing /// public class FMp4Encoder @@ -49,7 +29,7 @@ namespace JT1078.FMp4 FMp4Constants.TFHD_FLAG_DEFAULT_FLAGS; const uint TrunFlags = FMp4Constants.TRUN_FLAG_DATA_OFFSET_PRESENT | - FMp4Constants.TRUN_FLAG_SAMPLE_SIZE_PRESENT| + FMp4Constants.TRUN_FLAG_SAMPLE_SIZE_PRESENT | FMp4Constants.TRUN_FLAG_FIRST_SAMPLE_FLAGS_PRESENT; const uint SampleDescriptionIndex = 1; @@ -325,10 +305,9 @@ namespace JT1078.FMp4 /// 编码首帧 /// ftyp moov /// - /// - /// + /// /// - public byte[] FirstVideoBox(in H264NALU sps, in H264NALU pps) + public byte[] FirstVideoBox(in JT1078AVFrame avframe) { byte[] buffer = FMp4ArrayPool.Rent(CacheSize); FMp4MessagePackWriter writer = new FMp4MessagePackWriter(buffer); @@ -348,8 +327,6 @@ namespace JT1078.FMp4 fileTypeBox.CompatibleBrands.Add("iso6"); fileTypeBox.ToBuffer(ref writer); - ExpGolombReader h264GolombReader = new ExpGolombReader(sps.RawData); - var spsInfo = h264GolombReader.ReadSPS(); //moov MovieBox movieBox = new MovieBox(); movieBox.MovieHeaderBox = new MovieHeaderBox(0, 2); @@ -366,8 +343,8 @@ namespace JT1078.FMp4 movieBox.TrackBox.TrackHeaderBox.TrackID = VideoTrackID; movieBox.TrackBox.TrackHeaderBox.Duration = 0; movieBox.TrackBox.TrackHeaderBox.TrackIsAudio = false; - movieBox.TrackBox.TrackHeaderBox.Width = (uint)spsInfo.width; - movieBox.TrackBox.TrackHeaderBox.Height = (uint)spsInfo.height; + movieBox.TrackBox.TrackHeaderBox.Width = (uint)avframe.Width; + movieBox.TrackBox.TrackHeaderBox.Height = (uint)avframe.Height; movieBox.TrackBox.MediaBox = new MediaBox(); movieBox.TrackBox.MediaBox.MediaHeaderBox = new MediaHeaderBox(); movieBox.TrackBox.MediaBox.MediaHeaderBox.CreationTime = 0; @@ -386,16 +363,19 @@ namespace JT1078.FMp4 movieBox.TrackBox.MediaBox.MediaInformationBox.SampleTableBox = new SampleTableBox(); movieBox.TrackBox.MediaBox.MediaInformationBox.SampleTableBox.SampleDescriptionBox = new SampleDescriptionBox(); movieBox.TrackBox.MediaBox.MediaInformationBox.SampleTableBox.SampleDescriptionBox.SampleEntries = new List(); - //h264 + //h264 + //profileIdc profileCompat levelIdc + //0x64 0x00 0x1e + //avc1.64001e AVC1SampleEntry avc1 = new AVC1SampleEntry(); avc1.AVCConfigurationBox = new AVCConfigurationBox(); avc1.Width = (ushort)movieBox.TrackBox.TrackHeaderBox.Width; avc1.Height = (ushort)movieBox.TrackBox.TrackHeaderBox.Height; - avc1.AVCConfigurationBox.AVCLevelIndication = spsInfo.levelIdc; - avc1.AVCConfigurationBox.AVCProfileIndication = spsInfo.profileIdc; - avc1.AVCConfigurationBox.ProfileCompatibility = (byte)spsInfo.profileCompat; - avc1.AVCConfigurationBox.PPSs = new List() { pps.RawData }; - avc1.AVCConfigurationBox.SPSs = new List() { sps.RawData }; + avc1.AVCConfigurationBox.AVCLevelIndication = avframe.LevelIdc; + avc1.AVCConfigurationBox.AVCProfileIndication = avframe.ProfileIdc; + avc1.AVCConfigurationBox.ProfileCompatibility = avframe.ProfileCompat; + avc1.AVCConfigurationBox.PPSs = new List() { avframe.PPS.RawData }; + avc1.AVCConfigurationBox.SPSs = new List() { avframe.SPS.RawData }; movieBox.TrackBox.MediaBox.MediaInformationBox.SampleTableBox.SampleDescriptionBox.SampleEntries.Add(avc1); movieBox.TrackBox.MediaBox.MediaInformationBox.SampleTableBox.TimeToSampleBox = new TimeToSampleBox(); movieBox.TrackBox.MediaBox.MediaInformationBox.SampleTableBox.SyncSampleBox = new SyncSampleBox(); @@ -430,7 +410,7 @@ namespace JT1078.FMp4 /// styp sidx moof mdat /// /// - public byte[] OtherVideoBox(in List nalus) + public byte[] OtherVideoBox(in List nalus, in string key,in bool keyframe) { byte[] buffer = FMp4ArrayPool.Rent(CacheSize); FMp4MessagePackWriter writer = new FMp4MessagePackWriter(buffer); @@ -450,32 +430,25 @@ namespace JT1078.FMp4 //mdat var mediaDataBox = new MediaDataBox(); - mediaDataBox.Data=new List(); - - string key = string.Empty; - bool keyFrame = nalus[0].NALUHeader.KeyFrame; + mediaDataBox.Data = new List(); + uint sampleSize = 0; var truns = new List(); - uint defaultSampleDuration = 0; foreach (var n in nalus) { - if (key==string.Empty) - { - key=n.GetKey(); - } - defaultSampleDuration+=DefaultSampleDuration; - truns.Add(new TrackRunBox.TrackRunInfo() - { - SampleDuration = DefaultSampleDuration, - SampleSize = (uint)(n.RawData.Length+n.StartCodePrefix.Length), - }); + sampleSize += (uint)(n.RawData.Length + n.StartCodePrefix.Length); mediaDataBox.Data.Add(n.RawData); } + truns.Add(new TrackRunBox.TrackRunInfo() + { + SampleDuration = DefaultSampleDuration, + SampleSize = sampleSize, + }); if (!TrackInfos.TryGetValue(key, out TrackInfo trackInfo)) { - trackInfo = new TrackInfo { SN = 1, SubsegmentDuration=0 }; + trackInfo = new TrackInfo { SN = 1, SubsegmentDuration = 0 }; TrackInfos.Add(key, trackInfo); } - if (trackInfo.SN == uint.MaxValue) + if (trackInfo.SN == 0) { trackInfo.SN = 1; } @@ -487,7 +460,7 @@ namespace JT1078.FMp4 { new SegmentIndexBox.SegmentIndex { - SubsegmentDuration=defaultSampleDuration + SubsegmentDuration=DefaultSampleDuration } }; segmentIndexBox.ToBuffer(ref writer); @@ -501,31 +474,36 @@ namespace JT1078.FMp4 movieFragmentBox.TrackFragmentBox.TrackFragmentHeaderBox = new TrackFragmentHeaderBox(TfhdFlags); movieFragmentBox.TrackFragmentBox.TrackFragmentHeaderBox.TrackID = VideoTrackID; movieFragmentBox.TrackFragmentBox.TrackFragmentHeaderBox.SampleDescriptionIndex = SampleDescriptionIndex; - movieFragmentBox.TrackFragmentBox.TrackFragmentHeaderBox.DefaultSampleSize = truns[0].SampleSize; - movieFragmentBox.TrackFragmentBox.TrackFragmentHeaderBox.DefaultSampleDuration = truns[0].SampleDuration; + movieFragmentBox.TrackFragmentBox.TrackFragmentHeaderBox.DefaultSampleSize = sampleSize; + movieFragmentBox.TrackFragmentBox.TrackFragmentHeaderBox.DefaultSampleDuration = DefaultSampleDuration; movieFragmentBox.TrackFragmentBox.TrackFragmentHeaderBox.DefaultSampleFlags = FMp4Constants.TFHD_FLAG_VIDEO_TPYE; movieFragmentBox.TrackFragmentBox.TrackFragmentBaseMediaDecodeTimeBox = new TrackFragmentBaseMediaDecodeTimeBox(); movieFragmentBox.TrackFragmentBox.TrackFragmentBaseMediaDecodeTimeBox.BaseMediaDecodeTime = trackInfo.SubsegmentDuration; //update trackInfo - trackInfo.SubsegmentDuration+=defaultSampleDuration; + trackInfo.SubsegmentDuration += DefaultSampleDuration; trackInfo.SN++; TrackInfos[key] = trackInfo; //trun - - movieFragmentBox.TrackFragmentBox.TrackRunBox = new TrackRunBox(1, flags: TrunFlags); + if (keyframe) + { + movieFragmentBox.TrackFragmentBox.TrackRunBox = new TrackRunBox(1, flags: + FMp4Constants.TRUN_FLAG_DATA_OFFSET_PRESENT | + FMp4Constants.TRUN_FLAG_FIRST_SAMPLE_FLAGS_PRESENT); + } + else + { + movieFragmentBox.TrackFragmentBox.TrackRunBox = new TrackRunBox(1, FMp4Constants.TRUN_FLAG_DATA_OFFSET_PRESENT); + } movieFragmentBox.TrackFragmentBox.TrackRunBox.FirstSampleFlags = FMp4Constants.TREX_FLAG_SAMPLE_DEPENDS_ON_I_PICTURE; movieFragmentBox.TrackFragmentBox.TrackRunBox.TrackRunInfos = truns; movieFragmentBox.ToBuffer(ref writer); - //mdat mediaDataBox.ToBuffer(ref writer); - var current2 = writer.GetCurrentPosition(); foreach (var postion in segmentIndexBox.ReferencedSizePositions) { writer.WriteUInt32Return((uint)(current2 - current1), postion); } - var data = writer.FlushAndGetArray(); return data; } diff --git a/src/JT1078.FMp4/JT1078.FMp4.xml b/src/JT1078.FMp4/JT1078.FMp4.xml index fead006..7a62a2b 100644 --- a/src/JT1078.FMp4/JT1078.FMp4.xml +++ b/src/JT1078.FMp4/JT1078.FMp4.xml @@ -1348,26 +1348,6 @@ FMp4编码 fmp4 stream data - ftyp - moov - mvex - trex Video - trex Audio - styp 1 - sidx 1 Video - sidx 1 Audio - moof 1 - traf 1 Video - traf 1 Audio - mdat 1 - ... - styp n - sidx 1 Video - sidx n Audio - moof n - traf n Video - traf n Audio - mdat n ref: https://www.w3.org/TR/mse-byte-stream-format-isobmff/#movie-fragment-relative-addressing @@ -1404,16 +1384,15 @@ - + 编码首帧 ftyp moov - - + - + 编码其他视频数据盒子 styp sidx moof mdat diff --git a/src/JT1078.Flv.Test/JT1078.Flv.Test.csproj b/src/JT1078.Flv.Test/JT1078.Flv.Test.csproj index 86f2a7d..d12783d 100644 --- a/src/JT1078.Flv.Test/JT1078.Flv.Test.csproj +++ b/src/JT1078.Flv.Test/JT1078.Flv.Test.csproj @@ -5,13 +5,13 @@ - + - + all runtime; build; native; contentfiles; analyzers; buildtransitive - + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/src/JT1078.Flv/FlvEncoder.cs b/src/JT1078.Flv/FlvEncoder.cs index dd949e8..7f5c24a 100644 --- a/src/JT1078.Flv/FlvEncoder.cs +++ b/src/JT1078.Flv/FlvEncoder.cs @@ -68,6 +68,7 @@ namespace JT1078.Flv /// 是否有音频 /// 帧率 默认25d 即每秒25帧 /// + [Obsolete("use EncoderScriptTag(JT1078AVFrame avframe, bool hasAudio = false, double frameRate = 25d)")] public byte[] EncoderScriptTag(SPSInfo spsInfo, bool hasAudio = false, double frameRate = 25d) { byte[] buffer = FlvArrayPool.Rent(1024); @@ -114,6 +115,60 @@ namespace JT1078.Flv } } + /// + /// 编码脚本Tag + /// + /// 解析后的av信息 + /// 是否有音频 + /// 帧率 默认25d 即每秒25帧 + /// + + public byte[] EncoderScriptTag(JT1078AVFrame avframe, bool hasAudio = false, double frameRate = 25d) + { + byte[] buffer = FlvArrayPool.Rent(1024); + try + { + FlvMessagePackWriter flvMessagePackWriter = new FlvMessagePackWriter(buffer); + //flv body script tag + //flv body tag header + FlvTags flvTags = new FlvTags + { + Type = TagType.ScriptData, + //flv body tag body + DataTagsData = new Amf3 + { + Amf3Metadatas = new List + { + new Amf3Metadata_Duration{Value = 0d}, + new Amf3Metadata_VideoDataRate{Value = 0d}, + new Amf3Metadata_VideoCodecId{Value = 7d}, + new Amf3Metadata_FrameRate{Value = frameRate}, + new Amf3Metadata_Width(){ + Value=avframe.Width + }, + new Amf3Metadata_Height(){ + Value=avframe.Height + }, + } + } + }; + if (hasAudio) + { + flvTags.DataTagsData.Amf3Metadatas.Add(new Amf3Metadata_AudioCodecId()); + flvTags.DataTagsData.Amf3Metadatas.Add(new Amf3Metadata_AudioSampleRate()); + flvTags.DataTagsData.Amf3Metadatas.Add(new Amf3Metadata_AudioSampleSize()); + flvTags.DataTagsData.Amf3Metadatas.Add(new Amf3Metadata_AudioStereo()); + } + flvMessagePackWriter.WriteFlvTag(flvTags); + flvMessagePackWriter.WriteUInt32((uint)(flvTags.DataSize + 11)); + return flvMessagePackWriter.FlushAndGetArray(); + } + finally + { + FlvArrayPool.Return(buffer); + } + } + /// /// 编码首帧视频，即videoTag[0] /// @@ -122,6 +177,7 @@ namespace JT1078.Flv /// /// /// + [Obsolete("use EncoderFirstVideoTag(JT1078AVFrame avframe)")] public byte[] EncoderFirstVideoTag(SPSInfo spsInfo, H264NALU sps, H264NALU pps, H264NALU sei) { byte[] buffer = FlvArrayPool.Rent(4096); @@ -165,6 +221,54 @@ namespace JT1078.Flv } } + /// + /// 编码首帧视频，即videoTag[0] + /// + /// + /// + public byte[] EncoderFirstVideoTag(JT1078AVFrame avframe) + { + byte[] buffer = FlvArrayPool.Rent(4096); + try + { + FlvMessagePackWriter flvMessagePackWriter = new FlvMessagePackWriter(buffer); + //flv body video tag + //flv body tag header + FlvTags flvTags = new FlvTags + { + Type = TagType.Video, + Timestamp = (uint)avframe.SPS.Timestamp, + TimestampExt = 0, + StreamId = 0, + //flv body tag body + VideoTagsData = new VideoTags() + }; + flvTags.VideoTagsData.FrameType = FrameType.KeyFrame; + flvTags.VideoTagsData.VideoData = new AvcVideoPacke + { + AvcPacketType = AvcPacketType.SequenceHeader, + CompositionTime = 0 + }; + AVCDecoderConfigurationRecord aVCDecoderConfigurationRecord = new AVCDecoderConfigurationRecord + { + AVCProfileIndication = avframe.ProfileIdc, + ProfileCompatibility = avframe.ProfileCompat, + AVCLevelIndication = avframe.LevelIdc, + NumOfPictureParameterSets = 1, + PPSBuffer = avframe.PPS.RawData, + SPSBuffer = avframe.SPS.RawData + }; + flvTags.VideoTagsData.VideoData.AVCDecoderConfiguration = aVCDecoderConfigurationRecord; + flvMessagePackWriter.WriteFlvTag(flvTags); + flvMessagePackWriter.WriteUInt32((uint)(flvTags.DataSize + 11)); + return flvMessagePackWriter.FlushAndGetArray(); + } + finally + { + FlvArrayPool.Return(buffer); + } + } + /// /// 编码首帧音频，即audioTag[0] /// diff --git a/src/JT1078.Flv/JT1078.Flv.xml b/src/JT1078.Flv/JT1078.Flv.xml index 6864acb..5580567 100644 --- a/src/JT1078.Flv/JT1078.Flv.xml +++ b/src/JT1078.Flv/JT1078.Flv.xml @@ -203,6 +203,15 @@ 帧率 默认25d 即每秒25帧 + + + 编码脚本Tag + + 解析后的av信息 + 是否有音频 + 帧率 默认25d 即每秒25帧 + + 编码首帧视频，即videoTag[0] @@ -213,6 +222,13 @@ + + + 编码首帧视频，即videoTag[0] + + + + 编码首帧音频，即audioTag[0] diff --git a/src/JT1078.Hls.Test/JT1078.Hls.Test.csproj b/src/JT1078.Hls.Test/JT1078.Hls.Test.csproj index 00eb0f7..70c50c5 100644 --- a/src/JT1078.Hls.Test/JT1078.Hls.Test.csproj +++ b/src/JT1078.Hls.Test/JT1078.Hls.Test.csproj @@ -15,14 +15,14 @@ - - + + - + all runtime; build; native; contentfiles; analyzers; buildtransitive - + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/src/JT1078.Protocol.Benchmark/JT1078.Protocol.Benchmark.csproj b/src/JT1078.Protocol.Benchmark/JT1078.Protocol.Benchmark.csproj index b05c7e4..115211a 100644 --- a/src/JT1078.Protocol.Benchmark/JT1078.Protocol.Benchmark.csproj +++ b/src/JT1078.Protocol.Benchmark/JT1078.Protocol.Benchmark.csproj @@ -12,7 +12,7 @@ - + diff --git a/src/JT1078.Protocol.Test/JT1078.Protocol.Test.csproj b/src/JT1078.Protocol.Test/JT1078.Protocol.Test.csproj index 291f022..26535c5 100644 --- a/src/JT1078.Protocol.Test/JT1078.Protocol.Test.csproj +++ b/src/JT1078.Protocol.Test/JT1078.Protocol.Test.csproj @@ -7,10 +7,10 @@ - - + + - + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/src/JT1078.Protocol/H264/H264Decoder.cs b/src/JT1078.Protocol/H264/H264Decoder.cs index d67ed35..7f7fbf6 100644 --- a/src/JT1078.Protocol/H264/H264Decoder.cs +++ b/src/JT1078.Protocol/H264/H264Decoder.cs @@ -1,4 +1,5 @@ -using System; +using JT1078.Protocol.MessagePack; +using System; using System.Collections.Concurrent; using System.Collections.Generic; using System.Linq; @@ -67,6 +68,99 @@ namespace JT1078.Protocol.H264 return h264NALUs; } + /// + /// + /// + /// + /// + /// + public JT1078AVFrame ParseAVFrame(JT1078Package package, string key = null) + { + JT1078AVFrame jT1078AVFrame = new JT1078AVFrame(); + jT1078AVFrame.LogicChannelNumber = package.LogicChannelNumber; + jT1078AVFrame.SIM = package.SIM; + jT1078AVFrame.Nalus = new List(); + int i = 0, state = 0, laststate = 0; + int? lastIndex = null; + int length = package.Bodies.Length; + byte value; + ReadOnlySpan buffer = package.Bodies; + while (i < length) + { + value = buffer[i++]; + switch (state) + { + case 0: + if (value == 0) state = 1; + break; + case 1: + state = value == 0 ? 2 : 0; + break; + case 2: + case 3: + if (value == 0) + { + state = 3; + } + else if (value == 1 && i < length) + { + if (lastIndex.HasValue) + { + var tmp = buffer.Slice(lastIndex.Value, i - state - 1 - lastIndex.Value); + H264NALU nalu = Create(package, tmp, state + 1); + if (nalu.NALUHeader.NalUnitType == NalUnitType.PPS) + { + jT1078AVFrame.PPS = nalu; + } + else if (nalu.NALUHeader.NalUnitType == NalUnitType.SPS) + { + jT1078AVFrame.SPS = nalu; + ExpGolombReader h264GolombReader = new ExpGolombReader(jT1078AVFrame.SPS.RawData); + var spsInfo = h264GolombReader.ReadSPS(); + jT1078AVFrame.Width = spsInfo.width; + jT1078AVFrame.Height = spsInfo.height; + jT1078AVFrame.LevelIdc= spsInfo.levelIdc; + jT1078AVFrame.ProfileIdc=spsInfo.profileIdc; + jT1078AVFrame.ProfileCompat=(byte)spsInfo.profileCompat; + } + jT1078AVFrame.Nalus.Add(nalu); + } + lastIndex = i; + laststate = state + 1; + state = 0; + } + else + { + state = 0; + } + break; + default: + break; + } + } + if (lastIndex.HasValue) + { + H264NALU nalu = Create(package, buffer.Slice(lastIndex.Value), laststate); + if(nalu.NALUHeader.NalUnitType== NalUnitType.PPS) + { + jT1078AVFrame.PPS = nalu; + } + else if(nalu.NALUHeader.NalUnitType == NalUnitType.SPS) + { + jT1078AVFrame.SPS = nalu; + ExpGolombReader h264GolombReader = new ExpGolombReader(jT1078AVFrame.SPS.RawData); + var spsInfo = h264GolombReader.ReadSPS(); + jT1078AVFrame.Width = spsInfo.width; + jT1078AVFrame.Height = spsInfo.height; + jT1078AVFrame.LevelIdc = spsInfo.levelIdc; + jT1078AVFrame.ProfileIdc = spsInfo.profileIdc; + jT1078AVFrame.ProfileCompat = (byte)spsInfo.profileCompat; + } + jT1078AVFrame.Nalus.Add(nalu); + } + return jT1078AVFrame; + } + /// /// /// diff --git a/src/JT1078.Protocol/H264/H264NALU.cs b/src/JT1078.Protocol/H264/H264NALU.cs index f85ee7f..d709a94 100644 --- a/src/JT1078.Protocol/H264/H264NALU.cs +++ b/src/JT1078.Protocol/H264/H264NALU.cs @@ -5,7 +5,7 @@ using System.Text; namespace JT1078.Protocol.H264 { - public class H264NALU + public class H264NALU: IJT1078AVKey { public readonly static byte[] Start1 = new byte[3] { 0, 0, 1 }; public readonly static byte[] Start2 = new byte[4] { 0, 0, 0, 1 }; @@ -44,6 +44,13 @@ namespace JT1078.Protocol.H264 /// 数据体 /// public byte[] RawData { get; set; } + + public string GetAVKey() + { + return $"{SIM}_{LogicChannelNumber.ToString()}"; + } + + [Obsolete("use GetAVKey")] public string GetKey() { return $"{SIM}_{LogicChannelNumber.ToString()}"; diff --git a/src/JT1078.Protocol/IJT1078AVKey.cs b/src/JT1078.Protocol/IJT1078AVKey.cs new file mode 100644 index 0000000..5dcc03e --- /dev/null +++ b/src/JT1078.Protocol/IJT1078AVKey.cs @@ -0,0 +1,13 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace JT1078.Protocol +{ + public interface IJT1078AVKey + { + string GetAVKey(); + } +} diff --git a/src/JT1078.Protocol/JT1078.Protocol.xml b/src/JT1078.Protocol/JT1078.Protocol.xml index 9b03473..6e7e1c8 100644 --- a/src/JT1078.Protocol/JT1078.Protocol.xml +++ b/src/JT1078.Protocol/JT1078.Protocol.xml @@ -122,6 +122,14 @@ + + + + + + + + @@ -181,6 +189,17 @@ 数据体 + + + 终端设备SIM卡号 + BCD[6] + + + + + 逻辑通道号 + + V - 2 - 固定为2 @@ -322,7 +341,7 @@ - 该帧与上一个关键帧之间的时间间隔，单位毫秒(ms), + 该帧与上一帧之间的时间间隔，单位毫秒(ms), 当数据类型为非视频帧时，则没有该字段 diff --git a/src/JT1078.Protocol/JT1078AVFrame.cs b/src/JT1078.Protocol/JT1078AVFrame.cs new file mode 100644 index 0000000..fa906fe --- /dev/null +++ b/src/JT1078.Protocol/JT1078AVFrame.cs @@ -0,0 +1,39 @@ +using JT1078.Protocol.H264; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace JT1078.Protocol +{ + public class JT1078AVFrame : IJT1078AVKey + { + /// + /// 终端设备SIM卡号 + /// BCD[6] + /// + public string SIM { get; set; } + /// + /// 逻辑通道号 + /// + public byte LogicChannelNumber { get; set; } + public List Nalus { get; set; } + public H264NALU SPS { get; set; } + public H264NALU PPS { get; set; } + public int Width { get; set; } + public int Height { get; set; } + string VideoType { get; set; } = "avc1"; + public byte ProfileIdc { get; set; } + public byte ProfileCompat { get; set; } + public byte LevelIdc { get; set; } + public string ToCodecs() + { + return $"{VideoType}.{ProfileIdc:x2}{ProfileCompat:x2}{LevelIdc:x2}"; + } + public string GetAVKey() + { + return $"{SIM}_{LogicChannelNumber.ToString()}"; + } + } +} diff --git a/src/JT1078.Protocol/JT1078Package.cs b/src/JT1078.Protocol/JT1078Package.cs index 92a7692..5e09c10 100644 --- a/src/JT1078.Protocol/JT1078Package.cs +++ b/src/JT1078.Protocol/JT1078Package.cs @@ -4,7 +4,7 @@ using System.Text; namespace JT1078.Protocol { - public class JT1078Package + public class JT1078Package: IJT1078AVKey { /// /// 帧头标识 @@ -74,7 +74,7 @@ namespace JT1078.Protocol /// public ushort LastIFrameInterval { get; set; } /// - /// 该帧与上一个关键帧之间的时间间隔，单位毫秒(ms), + /// 该帧与上一帧之间的时间间隔，单位毫秒(ms), /// 当数据类型为非视频帧时，则没有该字段 /// public ushort LastFrameInterval { get; set; } @@ -87,9 +87,15 @@ namespace JT1078.Protocol /// public byte[] Bodies{ get; set; } + [Obsolete("use GetAVKey()")] public string GetKey() { return $"{SIM}_{LogicChannelNumber.ToString()}"; } + + public string GetAVKey() + { + return $"{SIM}_{LogicChannelNumber.ToString()}"; + } } } diff --git a/src/JT1078.SignalR.Test/Services/ToWebSocketService.cs b/src/JT1078.SignalR.Test/Services/ToWebSocketService.cs index ddfe857..a4e58f6 100644 --- a/src/JT1078.SignalR.Test/Services/ToWebSocketService.cs +++ b/src/JT1078.SignalR.Test/Services/ToWebSocketService.cs @@ -18,6 +18,7 @@ using System.IO; using JT1078.Protocol.Extensions; using JT1078.Protocol.H264; using System.Net.WebSockets; +using JT1078.Protocol.Enums; namespace JT1078.SignalR.Test.Services { @@ -49,7 +50,7 @@ namespace JT1078.SignalR.Test.Services public List q = new List(); - public void a() + void Init() { List packages = new List(); var lines = File.ReadAllLines(Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "H264", "jt1078_6.txt")); @@ -66,12 +67,10 @@ namespace JT1078.SignalR.Test.Services } } - var nalus1 = h264Decoder.ParseNALU(packages[0]); - q.Add(fMp4Encoder.FirstVideoBox( - nalus1.FirstOrDefault(f => f.NALUHeader.NalUnitType == NalUnitType.SPS), - nalus1.FirstOrDefault(f => f.NALUHeader.NalUnitType == NalUnitType.PPS))); + var avframe = h264Decoder.ParseAVFrame(packages[0]); + q.Add(fMp4Encoder.FirstVideoBox(avframe)); - List stream = new List(); + Queue mp4Frames = new Queue(); List filter = new List(); filter.Add(NalUnitType.SEI); filter.Add(NalUnitType.PPS); @@ -80,47 +79,35 @@ namespace JT1078.SignalR.Test.Services foreach (var package in packages) { List h264NALUs = h264Decoder.ParseNALU(package); - if (h264NALUs!=null && h264NALUs.Count>0) + if (h264NALUs != null && h264NALUs.Count > 0) { - stream.AddRange(h264NALUs.Where(w => !filter.Contains(w.NALUHeader.NalUnitType))); + Mp4Frame mp4Frame = new Mp4Frame + { + Key = package.GetKey(), + KeyFrame = package.Label3.DataType == JT1078DataType.视频I帧 + }; + mp4Frame.NALUs = h264NALUs; + mp4Frames.Enqueue(mp4Frame); } } - - List tmp = new List(); - H264NALU prevNalu = null; - foreach (var item in stream) + while (mp4Frames.TryDequeue(out Mp4Frame frame)) { - if (item.NALUHeader.KeyFrame) - { - if (tmp.Count>0) - { - q.Add(fMp4Encoder.OtherVideoBox(tmp)); - tmp.Clear(); - } - tmp.Add(item); - q.Add(fMp4Encoder.OtherVideoBox(tmp)); - tmp.Clear(); - prevNalu=item; - continue; - } - if (prevNalu!=null) //第一帧I帧 - { - if (tmp.Count>1) - { - q.Add(fMp4Encoder.OtherVideoBox(tmp)); - tmp.Clear(); - } - tmp.Add(item); - } + q.Add(fMp4Encoder.OtherVideoBox(frame.NALUs, frame.Key, frame.KeyFrame)); } } + class Mp4Frame + { + public string Key { get; set; } + public bool KeyFrame { get; set; } + public List NALUs { get; set; } + } public Dictionary flag = new Dictionary(); protected async override Task ExecuteAsync(CancellationToken stoppingToken) { - a(); + Init(); while (!stoppingToken.IsCancellationRequested) { try @@ -149,7 +136,7 @@ namespace JT1078.SignalR.Test.Services { logger.LogError(ex,""); } - await Task.Delay(80); + await Task.Delay(60); } } }