From c1576f8224d473a79b4f9c292d6887a9f959deff Mon Sep 17 00:00:00 2001 From: "SmallChi(Koike)" <564952747@qq.com> Date: Fri, 6 Aug 2021 18:16:51 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BF=AE=E6=94=B9fmp4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../JT1078AVEncoderContext.cs | 2 +- src/JT1078.FMp4.Test/H264/index.html | 4 +- src/JT1078.FMp4.Test/JT1078ToFMp4Box_Test.cs | 23 +++++----- src/JT1078.FMp4/FMp4Encoder.cs | 43 ++++++++++++++++--- src/JT1078.FMp4/JT1078.FMp4.csproj | 2 +- src/JT1078.FMp4/JT1078.FMp4.xml | 12 ++++-- .../H264/H264DecoderTest.cs | 21 +++++++++ .../JT1078.Protocol.Test.csproj | 3 ++ .../JT1078.SignalR.Test.csproj | 3 ++ .../Services/ToWebSocketService.cs | 34 ++++++++++++--- 10 files changed, 118 insertions(+), 29 deletions(-) diff --git a/src/JT1078.AV.Benchmark/JT1078AVEncoderContext.cs b/src/JT1078.AV.Benchmark/JT1078AVEncoderContext.cs index 7b3e0b6..8d17f76 100644 --- a/src/JT1078.AV.Benchmark/JT1078AVEncoderContext.cs +++ b/src/JT1078.AV.Benchmark/JT1078AVEncoderContext.cs @@ -117,7 +117,7 @@ namespace JT1078.AV.Benchmark { for (var i = 0; i < N; i++) { - var buffer = fmp4Encoder.EncoderOtherVideoBox(FMp4H264NALUs); + var buffer = fmp4Encoder.OtherVideoBox(FMp4H264NALUs); } } } diff --git a/src/JT1078.FMp4.Test/H264/index.html b/src/JT1078.FMp4.Test/H264/index.html index 217e61e..bdd0e6f 100644 --- a/src/JT1078.FMp4.Test/H264/index.html +++ b/src/JT1078.FMp4.Test/H264/index.html @@ -173,10 +173,10 @@ ws.on("video", (message) => { var buff=base64ToArrayBuffer(message); //console.log(buff); - //putPacket(buff); + putPacket(buff); //先直接喂进去 //mvhd.duration 谷歌浏览器不会缓存 - source_buffer.appendBuffer(buff); + //source_buffer.appendBuffer(buff); }); ws.start().catch(err => console.error(err)); } diff --git a/src/JT1078.FMp4.Test/JT1078ToFMp4Box_Test.cs b/src/JT1078.FMp4.Test/JT1078ToFMp4Box_Test.cs index 54ecf5a..14d1bbe 100644 --- a/src/JT1078.FMp4.Test/JT1078ToFMp4Box_Test.cs +++ b/src/JT1078.FMp4.Test/JT1078ToFMp4Box_Test.cs @@ -449,12 +449,12 @@ namespace JT1078.FMp4.Test } using var fileStream = new FileStream(filepath, FileMode.OpenOrCreate, FileAccess.Write); - var ftyp = fMp4Encoder.EncoderFtypBox(); + var ftyp = fMp4Encoder.FtypBox(); fileStream.Write(ftyp); var iNalus = h264Decoder.ParseNALU(packages[0]); //判断第一帧是否关键帧 - var moov = fMp4Encoder.EncoderMoovBox( + var moov = fMp4Encoder.VideoMoovBox( iNalus.FirstOrDefault(f => f.NALUHeader.NalUnitType == NalUnitType.SPS), iNalus.FirstOrDefault(f => f.NALUHeader.NalUnitType == NalUnitType.PPS)); fileStream.Write(moov); @@ -474,7 +474,7 @@ namespace JT1078.FMp4.Test { if (nalus.Count > 0) { - var otherBuffer = fMp4Encoder.EncoderOtherVideoBox(nalus); + var otherBuffer = fMp4Encoder.OtherVideoBox(nalus); fileStream.Write(otherBuffer); nalus.Clear(); } @@ -498,12 +498,12 @@ namespace JT1078.FMp4.Test } using var fileStream = new FileStream(filepath, FileMode.OpenOrCreate, FileAccess.Write); - var ftyp = fMp4Encoder.EncoderFtypBox(); + var ftyp = fMp4Encoder.FtypBox(); fileStream.Write(ftyp); var iNalus = h264Decoder.ParseNALU(packages[0]); //判断第一帧是否关键帧 - var moov = fMp4Encoder.EncoderMoovBox( + var moov = fMp4Encoder.VideoMoovBox( iNalus.FirstOrDefault(f => f.NALUHeader.NalUnitType == NalUnitType.SPS), iNalus.FirstOrDefault(f => f.NALUHeader.NalUnitType == NalUnitType.PPS)); fileStream.Write(moov); @@ -516,7 +516,7 @@ namespace JT1078.FMp4.Test { if (nalus.Count > 0) { - var otherBuffer = fMp4Encoder.EncoderOtherVideoBox(nalus); + var otherBuffer = fMp4Encoder.OtherVideoBox(nalus); fileStream.Write(otherBuffer); nalus.Clear(); } @@ -525,7 +525,7 @@ namespace JT1078.FMp4.Test } if (nalus.Count > 0) { - var otherBuffer = fMp4Encoder.EncoderOtherVideoBox(nalus); + var otherBuffer = fMp4Encoder.OtherVideoBox(nalus); fileStream.Write(otherBuffer); nalus.Clear(); } @@ -545,13 +545,13 @@ namespace JT1078.FMp4.Test } using var fileStream = new FileStream(filepath, FileMode.OpenOrCreate, FileAccess.Write); - var ftyp = fMp4Encoder.EncoderFtypBox(); + var ftyp = fMp4Encoder.FtypBox(); fileStream.Write(ftyp); var iPackage = packages.FirstOrDefault(f => f.Label3.DataType == JT1078DataType.视频I帧); var iNalus = h264Decoder.ParseNALU(iPackage); //判断第一帧是否关键帧 - var moov = fMp4Encoder.EncoderMoovBox( + var moov = fMp4Encoder.VideoMoovBox( iNalus.FirstOrDefault(f => f.NALUHeader.NalUnitType == NalUnitType.SPS), iNalus.FirstOrDefault(f => f.NALUHeader.NalUnitType == NalUnitType.PPS)); fileStream.Write(moov); @@ -564,7 +564,8 @@ namespace JT1078.FMp4.Test { if (nalus.Count > 0) { - var otherBuffer = fMp4Encoder.EncoderOtherVideoBox(nalus); + fileStream.Write(fMp4Encoder.StypBox()); + var otherBuffer = fMp4Encoder.OtherVideoBox(nalus); fileStream.Write(otherBuffer); nalus.Clear(); } @@ -573,7 +574,7 @@ namespace JT1078.FMp4.Test } if (nalus.Count > 0) { - var otherBuffer = fMp4Encoder.EncoderOtherVideoBox(nalus); + var otherBuffer = fMp4Encoder.OtherVideoBox(nalus); fileStream.Write(otherBuffer); nalus.Clear(); } diff --git a/src/JT1078.FMp4/FMp4Encoder.cs b/src/JT1078.FMp4/FMp4Encoder.cs index 08840dc..6c613db 100644 --- a/src/JT1078.FMp4/FMp4Encoder.cs +++ b/src/JT1078.FMp4/FMp4Encoder.cs @@ -32,10 +32,11 @@ namespace JT1078.FMp4 { Dictionary TrackInfos; - const uint DefaultSampleDuration = 48000u; + const uint DefaultSampleDuration = 40u; const uint DefaultSampleFlags = 0x1010000; const uint FirstSampleFlags = 33554432; const uint TfhdFlags = 0x2003a; + //const uint TrunFlags = 0x205; const uint TrunFlags = 0x205; const uint SampleDescriptionIndex = 1; const uint TrackID = 1; @@ -52,7 +53,7 @@ namespace JT1078.FMp4 /// 编码ftyp盒子 /// /// - public byte[] EncoderFtypBox() + public byte[] FtypBox() { byte[] buffer = FMp4ArrayPool.Rent(1024); FMp4MessagePackWriter writer = new FMp4MessagePackWriter(buffer); @@ -82,7 +83,7 @@ namespace JT1078.FMp4 /// 编码moov盒子 /// /// - public byte[] EncoderMoovBox(in H264NALU sps, in H264NALU pps) + public byte[] VideoMoovBox(in H264NALU sps, in H264NALU pps) { byte[] buffer = FMp4ArrayPool.Rent(sps.RawData.Length + pps.RawData.Length + 1024); FMp4MessagePackWriter writer = new FMp4MessagePackWriter(buffer); @@ -111,7 +112,8 @@ namespace JT1078.FMp4 movieBox.TrackBox.MediaBox.MediaHeaderBox = new MediaHeaderBox(); movieBox.TrackBox.MediaBox.MediaHeaderBox.CreationTime = 0; movieBox.TrackBox.MediaBox.MediaHeaderBox.ModificationTime = 0; - movieBox.TrackBox.MediaBox.MediaHeaderBox.Timescale = 1200000; + //movieBox.TrackBox.MediaBox.MediaHeaderBox.Timescale = 1200000; + movieBox.TrackBox.MediaBox.MediaHeaderBox.Timescale = 1000; movieBox.TrackBox.MediaBox.MediaHeaderBox.Duration = 0; movieBox.TrackBox.MediaBox.HandlerBox = new HandlerBox(); movieBox.TrackBox.MediaBox.HandlerBox.HandlerType = HandlerType.vide; @@ -160,11 +162,40 @@ namespace JT1078.FMp4 } } + /// + /// styp + /// + /// + public byte[] StypBox() + { + byte[] buffer = FMp4ArrayPool.Rent(1024); + FMp4MessagePackWriter writer = new FMp4MessagePackWriter(buffer); + try + { + SegmentTypeBox stypTypeBox = new SegmentTypeBox(); + stypTypeBox.MajorBrand = "isom"; + stypTypeBox.MinorVersion = "\0\0\0\0"; + stypTypeBox.CompatibleBrands.Add("isom"); + stypTypeBox.CompatibleBrands.Add("mp42"); + stypTypeBox.CompatibleBrands.Add("msdh"); + stypTypeBox.CompatibleBrands.Add("msix"); + stypTypeBox.CompatibleBrands.Add("iso5"); + stypTypeBox.CompatibleBrands.Add("iso6"); + stypTypeBox.ToBuffer(ref writer); + var data = writer.FlushAndGetArray(); + return data; + } + finally + { + FMp4ArrayPool.Return(buffer); + } + } + /// /// 编码其他视频数据盒子 /// /// - public byte[] EncoderOtherVideoBox(in List nalus) + public byte[] OtherVideoBox(in List nalus) { byte[] buffer = FMp4ArrayPool.Rent(nalus.Sum(s => s.RawData.Length + s.StartCodePrefix.Length) + 4096); FMp4MessagePackWriter writer = new FMp4MessagePackWriter(buffer); @@ -193,12 +224,14 @@ namespace JT1078.FMp4 { truns.Add(new TrackRunBox.TrackRunInfo() { + SampleDuration=40, SampleSize = iSize, }); iSize = 0; } truns.Add(new TrackRunBox.TrackRunInfo() { + SampleDuration = 40, SampleSize = (uint)(nalu.RawData.Length + nalu.StartCodePrefix.Length), }); } diff --git a/src/JT1078.FMp4/JT1078.FMp4.csproj b/src/JT1078.FMp4/JT1078.FMp4.csproj index 9f66ea3..841bb84 100644 --- a/src/JT1078.FMp4/JT1078.FMp4.csproj +++ b/src/JT1078.FMp4/JT1078.FMp4.csproj @@ -14,7 +14,7 @@ https://github.com/SmallChi/JT1078/blob/master/LICENSE https://github.com/SmallChi/JT1078/blob/master/LICENSE true - 1.0.0-preview4 + 1.0.0-preview5 false true LICENSE diff --git a/src/JT1078.FMp4/JT1078.FMp4.xml b/src/JT1078.FMp4/JT1078.FMp4.xml index b8e311b..0c58fb6 100644 --- a/src/JT1078.FMp4/JT1078.FMp4.xml +++ b/src/JT1078.FMp4/JT1078.FMp4.xml @@ -1325,19 +1325,25 @@ - + 编码ftyp盒子 - + 编码moov盒子 - + + + styp + + + + 编码其他视频数据盒子 diff --git a/src/JT1078.Protocol.Test/H264/H264DecoderTest.cs b/src/JT1078.Protocol.Test/H264/H264DecoderTest.cs index 4cdf40b..25d1456 100644 --- a/src/JT1078.Protocol.Test/H264/H264DecoderTest.cs +++ b/src/JT1078.Protocol.Test/H264/H264DecoderTest.cs @@ -116,5 +116,26 @@ namespace JT1078.Protocol.Test.H264 } fileStream.Close(); } + + [Fact] + public void ParseNALUTest5() + { + string file = "jt1078_6"; + var lines = File.ReadAllLines(Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "H264", $"{file}.txt")); + List nALUs = new List(); + H264Decoder decoder = new H264Decoder(); + foreach (var line in lines) + { + var bytes = line.ToHexBytes(); + JT1078Package package = JT1078Serializer.Deserialize(bytes); + var packageMerge = JT1078Serializer.Merge(package); + if (packageMerge != null) + { + var nalus = decoder.ParseNALU(packageMerge); + nALUs = nALUs.Concat(nalus).ToList(); + } + } + var a = nALUs.Count(c => !c.Slice); + } } } diff --git a/src/JT1078.Protocol.Test/JT1078.Protocol.Test.csproj b/src/JT1078.Protocol.Test/JT1078.Protocol.Test.csproj index 85d4d00..3edd26f 100644 --- a/src/JT1078.Protocol.Test/JT1078.Protocol.Test.csproj +++ b/src/JT1078.Protocol.Test/JT1078.Protocol.Test.csproj @@ -32,5 +32,8 @@ Always + + Always + diff --git a/src/JT1078.SignalR.Test/JT1078.SignalR.Test.csproj b/src/JT1078.SignalR.Test/JT1078.SignalR.Test.csproj index fcab1c5..9860de1 100644 --- a/src/JT1078.SignalR.Test/JT1078.SignalR.Test.csproj +++ b/src/JT1078.SignalR.Test/JT1078.SignalR.Test.csproj @@ -23,5 +23,8 @@ Always + + Always + diff --git a/src/JT1078.SignalR.Test/Services/ToWebSocketService.cs b/src/JT1078.SignalR.Test/Services/ToWebSocketService.cs index f2e0b39..36c465f 100644 --- a/src/JT1078.SignalR.Test/Services/ToWebSocketService.cs +++ b/src/JT1078.SignalR.Test/Services/ToWebSocketService.cs @@ -53,15 +53,19 @@ namespace JT1078.SignalR.Test.Services { List packages = new List(); //var lines = File.ReadAllLines(Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "H264", "jt1078_3.txt")); - var lines = File.ReadAllLines(Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "H264", "jt1078_5.txt")); + //var lines = File.ReadAllLines(Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "H264", "jt1078_5.txt")); + var lines = File.ReadAllLines(Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "H264", "jt1078_6.txt")); int mergeBodyLength = 0; foreach (var line in lines) { - var data = line.Split(','); + //var data = line.Split(','); //jt1078_5 - var bytes = data[1].ToHexBytes(); + //var bytes = data[1].ToHexBytes(); //jt1078_3 //var bytes = data[6].ToHexBytes(); + //jt1078_6 + var bytes = line.ToHexBytes(); + JT1078Package package = JT1078Serializer.Deserialize(bytes); mergeBodyLength += package.DataBodyLength; var packageMerge = JT1078Serializer.Merge(package); @@ -74,12 +78,12 @@ namespace JT1078.SignalR.Test.Services //var styp = fMp4Encoder.EncoderStypBox(); //first.Add(styp); //q.Enqueue(styp); - var ftyp = fMp4Encoder.EncoderFtypBox(); + var ftyp = fMp4Encoder.FtypBox(); //q.Enqueue(ftyp); first.Add(ftyp); var package1 = packages[0]; var nalus1 = h264Decoder.ParseNALU(package1); - var moov = fMp4Encoder.EncoderMoovBox( + var moov = fMp4Encoder.VideoMoovBox( nalus1.FirstOrDefault(f => f.NALUHeader.NalUnitType == NalUnitType.SPS), nalus1.FirstOrDefault(f => f.NALUHeader.NalUnitType == NalUnitType.PPS)); //q.Enqueue(moov); @@ -90,6 +94,23 @@ namespace JT1078.SignalR.Test.Services foreach (var package in packages) { List h264NALUs = h264Decoder.ParseNALU(package); + //if(package.Label3.DataType== Protocol.Enums.JT1078DataType.视频I帧) + //{ + // if (nalus.Count > 0) + // { + // var otherBuffer = fMp4Encoder.OtherVideoBox(nalus); + // q.Add(fMp4Encoder.StypBox().Concat(otherBuffer).ToArray()); + // nalus.Clear(); + // } + // else + // { + // nalus = nalus.Concat(h264NALUs).ToList(); + // } + //} + //else + //{ + // nalus = nalus.Concat(h264NALUs).ToList(); + //} foreach (var nalu in h264NALUs) { if (nalu.Slice) @@ -101,7 +122,8 @@ namespace JT1078.SignalR.Test.Services { if (nalus.Count > 0) { - var otherBuffer = fMp4Encoder.EncoderOtherVideoBox(nalus); + q.Add(fMp4Encoder.StypBox()); + var otherBuffer = fMp4Encoder.OtherVideoBox(nalus); q.Add(otherBuffer); nalus.Clear(); }