diff --git a/doc/ffmpeginfo.txt b/doc/ffmpeginfo.txt index 5b1e5cd..2d21748 100644 --- a/doc/ffmpeginfo.txt +++ b/doc/ffmpeginfo.txt @@ -2,4 +2,6 @@ ffmpeg -i demo.mp4 -c copy -f flv -vcodec h264 -acodec aac demo_flv.flv ./ffmpeg -i JT1078_3.h264 -r 25 -c copy -f flv "D:\JT1078_3.flv" -ffmpeg -f dshow -i video="USB2.0 PC CAMERA" -t 60 -c copy -f h264 -vcodec h264 jt1078.264 \ No newline at end of file +ffmpeg -f dshow -i video="USB2.0 PC CAMERA" -t 60 -c copy -f h264 -vcodec h264 jt1078.264 + +chrome://media-internals/ \ No newline at end of file diff --git a/src/JT1078.Flv.Test/FlvEncoderTest.cs b/src/JT1078.Flv.Test/FlvEncoderTest.cs index 2d23b9e..2c1029e 100644 --- a/src/JT1078.Flv.Test/FlvEncoderTest.cs +++ b/src/JT1078.Flv.Test/FlvEncoderTest.cs @@ -243,16 +243,23 @@ namespace JT1078.Flv.Test } } } - //var tmp1 = h264NALULs.Where(w => w.NALUHeader.NalUnitType == 7).ToList(); + var tmp1 = h264NALULs.Where(w => w.NALUHeader.NalUnitType == 7).ToList(); List tmpSpss = new List(); List times = new List(); + List lastIFrameIntervals = new List(); + List lastFrameIntervals = new List(); List type = new List(); foreach (var item in h264NALULs) - { - //ExpGolombReader expGolombReader = new ExpGolombReader(item.RawData); + { //type.Add(item.NALUHeader.NalUnitType); times.Add(item.Timestamp); - //tmpSpss.Add(expGolombReader.ReadSPS()); + lastFrameIntervals.Add(item.LastFrameInterval); + lastIFrameIntervals.Add(item.LastIFrameInterval); + if(item.NALUHeader.NalUnitType == 7) + { + ExpGolombReader expGolombReader = new ExpGolombReader(item.RawData); + tmpSpss.Add(expGolombReader.ReadSPS()); + } } fileStream = new FileStream(filepath, FileMode.OpenOrCreate, FileAccess.Write); var totalPage = (h264NALULs.Count + 10 - 1) / 10; @@ -261,7 +268,7 @@ namespace JT1078.Flv.Test var flv2 = encoder.CreateFlvFrame(h264NALULs.Skip(i * 10).Take(10).ToList()); if (flv2.Length != 0) { - fileStream.Write(flv2); + //fileStream.Write(flv2); } } } @@ -301,7 +308,7 @@ namespace JT1078.Flv.Test FlvEncoder flvEncoder = new FlvEncoder(); string key = "test"; var bufferFlvFrame = new byte[] { 0xA, 0xB, 0xC, 0xD, 0xE, 0xF }; - FlvEncoder.FlvFirstFrameCache.TryAdd(key, (2, new byte[] { 1, 2, 3, 4, 5, 6 })); + FlvEncoder.FirstFlvFrameCache.TryAdd(key, (2, new byte[] { 1, 2, 3, 4, 5, 6 },true)); var buffer=flvEncoder.GetFirstFlvFrame(key, bufferFlvFrame); //替换PreviousTagSize 4位的长度为首帧的 PreviousTagSize Assert.Equal(new byte[] { 1, 2, 3, 4, 5, 6, 0, 0, 0, 2, 0xE, 0xF }, buffer); diff --git a/src/JT1078.Flv.Test/H264/index.html b/src/JT1078.Flv.Test/H264/index.html index 0a06bc3..1c804f0 100644 --- a/src/JT1078.Flv.Test/H264/index.html +++ b/src/JT1078.Flv.Test/H264/index.html @@ -9,28 +9,40 @@ diff --git a/src/JT1078.Flv/FlvEncoder.cs b/src/JT1078.Flv/FlvEncoder.cs index d8def8d..e33ded9 100644 --- a/src/JT1078.Flv/FlvEncoder.cs +++ b/src/JT1078.Flv/FlvEncoder.cs @@ -4,6 +4,7 @@ using JT1078.Flv.MessagePack; using JT1078.Flv.Metadata; using JT1078.Protocol; using JT1078.Protocol.Enums; +using Microsoft.Extensions.Logging; using System; using System.Buffers.Binary; using System.Collections.Concurrent; @@ -17,31 +18,29 @@ namespace JT1078.Flv { public class FlvEncoder { - public class FlvFrameInfo - { - public uint PreviousTagSize { get; set;} - public uint LastFrameInterval { get; set; } - public uint LastIFrameInterval { get; set; } - public ulong Timestamp { get; set; } - public uint Interval { get; set; } - public JT1078DataType DataType { get; set; } - } - public static readonly byte[] VideoFlvHeaderBuffer; private static readonly Flv.H264.H264Decoder H264Decoder; private static readonly ConcurrentDictionary VideoSPSDict; private static readonly ConcurrentDictionary FlvFrameInfoDict; - internal static readonly ConcurrentDictionary FlvFirstFrameCache; + internal static readonly ConcurrentDictionary FirstFlvFrameCache; + private readonly ILogger logger; static FlvEncoder() { FlvHeader VideoFlvHeader = new FlvHeader(true, false); VideoFlvHeaderBuffer = VideoFlvHeader.ToArray().ToArray(); VideoSPSDict = new ConcurrentDictionary(StringComparer.OrdinalIgnoreCase); FlvFrameInfoDict = new ConcurrentDictionary(StringComparer.OrdinalIgnoreCase); - FlvFirstFrameCache = new ConcurrentDictionary(StringComparer.OrdinalIgnoreCase); + FirstFlvFrameCache = new ConcurrentDictionary(StringComparer.OrdinalIgnoreCase); H264Decoder = new Flv.H264.H264Decoder(); } + public FlvEncoder() + { + } + public FlvEncoder(ILoggerFactory loggerFactory) + { + logger = loggerFactory.CreateLogger("FlvEncoder"); + } public byte[] CreateScriptTagFrame(int width, int height, double frameRate = 25d) { byte[] buffer = FlvArrayPool.Rent(1024); @@ -92,7 +91,7 @@ namespace JT1078.Flv } public byte[] CreateVideoTag0Frame(byte[] spsRawData, byte[] ppsRawData, SPSInfo spsInfo) { - byte[] buffer = FlvArrayPool.Rent(1024); + byte[] buffer = FlvArrayPool.Rent(2048); try { FlvMessagePackWriter flvMessagePackWriter = new FlvMessagePackWriter(buffer); @@ -125,6 +124,41 @@ namespace JT1078.Flv FlvArrayPool.Return(buffer); } } + public byte[] CreateSecondVideoTag0Frame(FlvFrameInfo flvFrameInfo,byte[] spsRawData, byte[] ppsRawData, SPSInfo spsInfo) + { + byte[] buffer = FlvArrayPool.Rent(2048); + try + { + FlvMessagePackWriter flvMessagePackWriter = new FlvMessagePackWriter(buffer); + //flv body video tag + //flv body tag header + FlvTags flvTags = new FlvTags(); + flvTags.Type = TagType.Video; + flvTags.Timestamp = flvFrameInfo.Interval; + flvTags.TimestampExt = 0; + flvTags.StreamId = 0; + //flv body tag body + flvTags.VideoTagsData = new VideoTags(); + flvTags.VideoTagsData.FrameType = FrameType.KeyFrame; + flvTags.VideoTagsData.VideoData = new AvcVideoPacke(); + flvTags.VideoTagsData.VideoData.AvcPacketType = AvcPacketType.SequenceHeader; + flvTags.VideoTagsData.VideoData.CompositionTime = flvFrameInfo.LastIFrameInterval; + AVCDecoderConfigurationRecord aVCDecoderConfigurationRecord = new AVCDecoderConfigurationRecord(); + aVCDecoderConfigurationRecord.AVCProfileIndication = spsInfo.profileIdc; + aVCDecoderConfigurationRecord.ProfileCompatibility = (byte)spsInfo.profileCompat; + aVCDecoderConfigurationRecord.AVCLevelIndication = spsInfo.levelIdc; + aVCDecoderConfigurationRecord.NumOfPictureParameterSets = 1; + aVCDecoderConfigurationRecord.PPSBuffer = ppsRawData; + aVCDecoderConfigurationRecord.SPSBuffer = spsRawData; + flvTags.VideoTagsData.VideoData.AVCDecoderConfiguration = aVCDecoderConfigurationRecord; + flvMessagePackWriter.WriteFlvTag(flvTags); + return flvMessagePackWriter.FlushAndGetArray(); + } + finally + { + FlvArrayPool.Return(buffer); + } + } public byte[] CreateVideoTagOtherFrame(FlvFrameInfo flvFrameInfo, H264NALU nALU, H264NALU sei) { byte[] buffer = FlvArrayPool.Rent(65535); @@ -135,6 +169,8 @@ namespace JT1078.Flv //flv body tag header FlvTags flvTags = new FlvTags(); flvTags.Type = TagType.Video; + //pts + flvTags.Timestamp = flvFrameInfo.Interval; flvTags.TimestampExt = 0; flvTags.StreamId = 0; //flv body tag body @@ -151,19 +187,21 @@ namespace JT1078.Flv } if(flvFrameInfo.DataType== JT1078DataType.视频I帧) { + //cts flvTags.VideoTagsData.VideoData.CompositionTime = flvFrameInfo.LastIFrameInterval; } else { + //cts flvTags.VideoTagsData.VideoData.CompositionTime = flvFrameInfo.LastFrameInterval; } - flvTags.Timestamp = flvFrameInfo.Interval; flvTags.VideoTagsData.VideoData.MultiData = new List(); flvTags.VideoTagsData.VideoData.MultiData.Add(nALU.RawData); - if(sei!=null && sei.RawData!=null && sei.RawData.Length > 0) - { - flvTags.VideoTagsData.VideoData.MultiData.Add(sei.RawData); - } + //忽略sei + //if (sei != null && sei.RawData != null && sei.RawData.Length > 0) + //{ + // flvTags.VideoTagsData.VideoData.MultiData.Add(sei.RawData); + //} flvMessagePackWriter.WriteFlvTag(flvTags); return flvMessagePackWriter.FlushAndGetArray(); } @@ -179,42 +217,83 @@ namespace JT1078.Flv { FlvMessagePackWriter flvMessagePackWriter = new FlvMessagePackWriter(buffer); H264NALU sps=null, pps=null, sei=null; - SPSInfo spsInfo = new SPSInfo(); foreach (var naln in nALUs) { string key = naln.GetKey(); if (sps != null && pps != null) { + var rawData = H264Decoder.DiscardEmulationPreventionBytes(sps.RawData); + ExpGolombReader h264GolombReader = new ExpGolombReader(rawData); + SPSInfo spsInfo = h264GolombReader.ReadSPS(); if (VideoSPSDict.TryGetValue(key, out var spsInfoCache)) { - //todo: 主次码流 - //if (spsInfoCache.height != spsInfo.height && spsInfoCache.width != spsInfo.width) - //{ - // if (FlvFrameInfoDict.TryGetValue(key, out FlvFrameInfo flvFrameInfo)) - // { - // var rawData = H264Decoder.DiscardEmulationPreventionBytes(sps.RawData); - // ExpGolombReader h264GolombReader = new ExpGolombReader(rawData); - // spsInfo = h264GolombReader.ReadSPS(); - // CreateFlvKeyFrame(ref flvMessagePackWriter, key, sps.RawData, pps.RawData, spsInfo); - // VideoSPSDict.TryUpdate(key, spsInfo, spsInfo); - // flvFrameInfo.LastIFrameInterval = 0; - // FlvFrameInfoDict.TryUpdate(key, flvFrameInfo, flvFrameInfo); - // } - //} + //切换主次码流 + //根据宽高来判断 + if (spsInfoCache.height != spsInfo.height && spsInfoCache.width != spsInfo.width) + { + if (logger != null) + { + if (logger.IsEnabled(LogLevel.Debug)) + { + logger.LogDebug($"Cache:{spsInfoCache.height}-{spsInfoCache.width},Current:{spsInfo.height}-{spsInfo.width}"); + } + } + VideoSPSDict.TryUpdate(key, spsInfo, spsInfoCache); + if (FlvFrameInfoDict.TryGetValue(key, out FlvFrameInfo flvFrameInfo)) + { + flvFrameInfo.Timestamp = naln.Timestamp; + flvFrameInfo.LastIFrameInterval = naln.LastIFrameInterval; + flvFrameInfo.LastFrameInterval = naln.LastFrameInterval; + if (logger != null) + { + if (logger.IsEnabled(LogLevel.Debug)) + { + logger.LogDebug($"Cache:{spsInfoCache.height}-{spsInfoCache.width},Current:{spsInfo.height}-{spsInfo.width}"); + } + } + var secondFlvKeyFrame=CreateSecondFlvKeyFrame(sps.RawData, pps.RawData, spsInfo, flvFrameInfo); + flvMessagePackWriter.WriteArray(secondFlvKeyFrame.Buffer); + flvFrameInfo.PreviousTagSize = secondFlvKeyFrame.PreviousTagSize; + FlvFrameInfoDict.TryUpdate(key, flvFrameInfo, flvFrameInfo); + if(FirstFlvFrameCache.TryGetValue(key,out var firstFlvFrameCacche)) + { + FirstFlvFrameCache.TryUpdate(key, (secondFlvKeyFrame.PreviousTagSize, secondFlvKeyFrame.Buffer, true), firstFlvFrameCacche); + } + } + } } else { - var rawData = H264Decoder.DiscardEmulationPreventionBytes(sps.RawData); - ExpGolombReader h264GolombReader = new ExpGolombReader(rawData); - spsInfo = h264GolombReader.ReadSPS(); - flvMessagePackWriter.WriteArray(VideoFlvHeaderBuffer); - CreateFlvKeyFrame(ref flvMessagePackWriter, key, sps.RawData, pps.RawData, spsInfo); + var firstFlvKeyFrame = CreateFirstFlvKeyFrame(sps.RawData, pps.RawData, spsInfo); + flvMessagePackWriter.WriteArray(firstFlvKeyFrame.Buffer); + if (logger != null) + { + if (logger.IsEnabled(LogLevel.Debug)) + { + logger.LogDebug($"Current:{spsInfo.height}-{spsInfo.width}"); + } + } + //cache PreviousTagSize + FlvFrameInfoDict.TryAdd(key, new FlvFrameInfo{ + PreviousTagSize = firstFlvKeyFrame.PreviousTagSize, + Interval = (uint)(pps.Timestamp- sps.Timestamp), + Timestamp= pps.Timestamp, + } + ); + FirstFlvFrameCache.TryAdd(key, (firstFlvKeyFrame.PreviousTagSize, firstFlvKeyFrame.Buffer, false)); VideoSPSDict.TryAdd(key, spsInfo); } sps = null; pps = null; continue; } + if (logger != null) + { + if (logger.IsEnabled(LogLevel.Debug)) + { + logger.LogDebug($"NalUnitType:{naln.NALUHeader.NalUnitType}"); + } + } //7 8 6 5 1 1 1 1 7 8 6 5 1 1 1 1 1 7 8 6 5 1 1 1 1 1 switch (naln.NALUHeader.NalUnitType) { @@ -224,6 +303,7 @@ namespace JT1078.Flv { flvFrameInfo.LastIFrameInterval = naln.LastIFrameInterval; flvFrameInfo.LastFrameInterval = naln.LastFrameInterval; + //当前的1078包与上一包1078的时间戳相减再进行累加 uint interval = (uint)(naln.Timestamp - flvFrameInfo.Timestamp); flvFrameInfo.Interval += interval; flvFrameInfo.Timestamp = naln.Timestamp; @@ -257,30 +337,55 @@ namespace JT1078.Flv FlvArrayPool.Return(buffer); } } - private void CreateFlvKeyFrame(ref FlvMessagePackWriter flvMessagePackWriter, string key, byte[] spsRawData, byte[] ppsRawData, SPSInfo spsInfo) + public (byte[] Buffer, uint PreviousTagSize) CreateFirstFlvKeyFrame(byte[] spsRawData, byte[] ppsRawData, SPSInfo spsInfo) + { + byte[] buffer = FlvArrayPool.Rent(65535); + try + { + FlvMessagePackWriter flvMessagePackWriter = new FlvMessagePackWriter(buffer); + //flv body PreviousTagSize awalys 0 + flvMessagePackWriter.WriteUInt32(0); + //flv body script tag + var scriptTagFrameBuffer= CreateScriptTagFrame(spsInfo.width, spsInfo.height); + flvMessagePackWriter.WriteArray(scriptTagFrameBuffer); + //flv script tag PreviousTagSize + flvMessagePackWriter.WriteUInt32((uint)scriptTagFrameBuffer.Length); + //flv body video tag 0 + var videoTagFrame0Buffer= CreateVideoTag0Frame(spsRawData, ppsRawData, spsInfo); + flvMessagePackWriter.WriteArray(videoTagFrame0Buffer); + uint videoTag0PreviousTagSize = (uint)videoTagFrame0Buffer.Length; + return (flvMessagePackWriter.FlushAndGetArray(), videoTag0PreviousTagSize); + } + finally + { + FlvArrayPool.Return(buffer); + } + } + public (byte[] Buffer,uint PreviousTagSize) CreateSecondFlvKeyFrame(byte[] spsRawData, byte[] ppsRawData, SPSInfo spsInfo, FlvFrameInfo flvFrameInfo) { - //flv body PreviousTagSize awalys 0 - flvMessagePackWriter.WriteUInt32(0); - //flv body script tag - var scriptTagFrameBuffer= CreateScriptTagFrame(spsInfo.width, spsInfo.height); - flvMessagePackWriter.WriteArray(scriptTagFrameBuffer); - //flv script tag PreviousTagSize - flvMessagePackWriter.WriteUInt32((uint)scriptTagFrameBuffer.Length); - //flv body video tag 0 - var videoTagFrame0Buffer= CreateVideoTag0Frame(spsRawData, ppsRawData, spsInfo); - flvMessagePackWriter.WriteArray(videoTagFrame0Buffer); - uint videoTag0PreviousTagSize = (uint)videoTagFrame0Buffer.Length; - //cache PreviousTagSize - FlvFrameInfoDict.AddOrUpdate(key, new FlvFrameInfo { PreviousTagSize = videoTag0PreviousTagSize}, (a, b) => { - b.PreviousTagSize = videoTag0PreviousTagSize; - return b; - }); - var buffer = flvMessagePackWriter.FlushAndGetArray(); - FlvFirstFrameCache.AddOrUpdate(key,(videoTag0PreviousTagSize, buffer),(a,b)=> { - b.PreviousTagSize = videoTag0PreviousTagSize; - b.Buffer = buffer; - return b; - }); + byte[] buffer = FlvArrayPool.Rent(65535); + try + { + FlvMessagePackWriter flvMessagePackWriter = new FlvMessagePackWriter(buffer); + //flv body PreviousTagSize awalys 0 + //当首包的时候为 0 + //当切花主次码流的时候需要根据上一包的大小 + flvMessagePackWriter.WriteUInt32(flvFrameInfo.PreviousTagSize); + //flv body script tag + var scriptTagFrameBuffer = CreateScriptTagFrame(spsInfo.width, spsInfo.height); + flvMessagePackWriter.WriteArray(scriptTagFrameBuffer); + //flv script tag PreviousTagSize + flvMessagePackWriter.WriteUInt32((uint)scriptTagFrameBuffer.Length); + //flv body video tag 0 + var videoTagFrame0Buffer = CreateSecondVideoTag0Frame(flvFrameInfo, spsRawData, ppsRawData, spsInfo); + flvMessagePackWriter.WriteArray(videoTagFrame0Buffer); + uint videoTag0PreviousTagSize = (uint)videoTagFrame0Buffer.Length; + return (flvMessagePackWriter.FlushAndGetArray(), videoTag0PreviousTagSize); + } + finally + { + FlvArrayPool.Return(buffer); + } } public byte[] CreateFlvFrame(JT1078Package package,int minimumLength = 65535) { @@ -290,17 +395,32 @@ namespace JT1078.Flv } public byte[] GetFirstFlvFrame(string key,byte[] bufferFlvFrame) { - if (FlvFirstFrameCache.TryGetValue(key, out var firstBuffer)) + if (FirstFlvFrameCache.TryGetValue(key, out var firstBuffer)) { - var length = firstBuffer.Buffer.Length + bufferFlvFrame.Length; + var length = firstBuffer.Buffer.Length + bufferFlvFrame.Length + VideoFlvHeaderBuffer.Length; byte[] buffer = FlvArrayPool.Rent(length); try { Span tmp = buffer; - firstBuffer.Buffer.CopyTo(tmp); - BinaryPrimitives.WriteUInt32BigEndian(bufferFlvFrame, firstBuffer.PreviousTagSize); - bufferFlvFrame.CopyTo(tmp.Slice(firstBuffer.Buffer.Length)); - return tmp.Slice(0, length).ToArray(); + VideoFlvHeaderBuffer.CopyTo(tmp); + if (firstBuffer.Changed) + { + //新用户进来需要替换为首包的PreviousTagSize 0 + BinaryPrimitives.WriteUInt32BigEndian(firstBuffer.Buffer, 0); + firstBuffer.Buffer.CopyTo(tmp.Slice(VideoFlvHeaderBuffer.Length)); + //新用户进来需要替换为上一包的PreviousTagSize + BinaryPrimitives.WriteUInt32BigEndian(bufferFlvFrame, firstBuffer.PreviousTagSize); + bufferFlvFrame.CopyTo(tmp.Slice(VideoFlvHeaderBuffer.Length + firstBuffer.Buffer.Length)); + return tmp.Slice(0, length).ToArray(); + } + else + { + firstBuffer.Buffer.CopyTo(tmp.Slice(VideoFlvHeaderBuffer.Length)); + //新用户进来需要替换为首包的PreviousTagSize + BinaryPrimitives.WriteUInt32BigEndian(bufferFlvFrame, firstBuffer.PreviousTagSize); + bufferFlvFrame.CopyTo(tmp.Slice(VideoFlvHeaderBuffer.Length + firstBuffer.Buffer.Length)); + return tmp.Slice(0, length).ToArray(); + } } finally { @@ -310,4 +430,14 @@ namespace JT1078.Flv return default; } } + + public class FlvFrameInfo + { + public uint PreviousTagSize { get; set; } + public uint LastFrameInterval { get; set; } + public uint LastIFrameInterval { get; set; } + public ulong Timestamp { get; set; } + public uint Interval { get; set; } + public JT1078DataType DataType { get; set; } + } } diff --git a/src/JT1078.Flv/H264/H264NALU.cs b/src/JT1078.Flv/H264/H264NALU.cs index b366a4a..54d4e51 100644 --- a/src/JT1078.Flv/H264/H264NALU.cs +++ b/src/JT1078.Flv/H264/H264NALU.cs @@ -30,7 +30,7 @@ namespace JT1078.Flv.H264 /// public ushort LastIFrameInterval { get; set; } /// - /// 该帧与上一个关键帧之间的时间间隔，单位毫秒(ms), + /// 该帧与上一个帧之间的时间间隔，单位毫秒(ms), /// 当数据类型为非视频帧时，则没有该字段 /// public ushort LastFrameInterval { get; set; } diff --git a/src/JT1078.Flv/JT1078.Flv.csproj b/src/JT1078.Flv/JT1078.Flv.csproj index bd75d2b..183d2dc 100644 --- a/src/JT1078.Flv/JT1078.Flv.csproj +++ b/src/JT1078.Flv/JT1078.Flv.csproj @@ -37,4 +37,7 @@ + + + diff --git a/src/JT1078.Flv/MessagePack/EXPGolombReader.cs b/src/JT1078.Flv/MessagePack/EXPGolombReader.cs index 7c2cd46..8a0372e 100644 --- a/src/JT1078.Flv/MessagePack/EXPGolombReader.cs +++ b/src/JT1078.Flv/MessagePack/EXPGolombReader.cs @@ -311,7 +311,7 @@ namespace JT1078.Flv.MessagePack } } - public struct SPSInfo + public class SPSInfo { public byte profileIdc { get; set; } public byte levelIdc { get; set; }