From b82fe5bdaca49db6526343be8baa0ab28255953f Mon Sep 17 00:00:00 2001
From: smallchi <564952747@qq.com>
Date: Mon, 14 Oct 2019 18:37:38 +0800
Subject: [PATCH] =?UTF-8?q?1.=E5=A2=9E=E5=8A=A0flv=E7=BC=96=E7=A0=81?=
 =?UTF-8?q?=E5=99=A8=E6=94=AF=E6=8C=81=E5=A4=9A=E4=B8=AANALU(SEI=E5=8C=85)?=
 =?UTF-8?q?=202.=E5=9C=A8flv=E7=BC=96=E7=A0=81=E4=B8=AD=E5=B0=861078?=
 =?UTF-8?q?=E5=AF=B9=E5=BA=94=E7=9A=84=E5=8F=82=E6=95=B0=E7=BB=99=E4=BD=BF?=
 =?UTF-8?q?=E7=94=A8=E4=B8=8A?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 src/JT1078.Flv.Test/FlvEncoderTest.cs         | 29 +++----
 src/JT1078.Flv.Test/H264/index.html           | 13 +++-
 src/JT1078.Flv.Test/JT1078.Flv.Test.csproj    |  3 +
 src/JT1078.Flv/FlvEncoder.cs                  | 75 ++++++++++++-------
 src/JT1078.Flv/H264/H264Decoder.cs            |  1 +
 src/JT1078.Flv/H264/H264NALU.cs               |  6 ++
 .../MessagePack/FlvMessagePackWriter_Flv.cs   | 10 ++-
 src/JT1078.Flv/Metadata/AvcVideoPacke.cs      |  3 +-
 8 files changed, 85 insertions(+), 55 deletions(-)

diff --git a/src/JT1078.Flv.Test/FlvEncoderTest.cs b/src/JT1078.Flv.Test/FlvEncoderTest.cs
index f68fdfc..2d23b9e 100644
--- a/src/JT1078.Flv.Test/FlvEncoderTest.cs
+++ b/src/JT1078.Flv.Test/FlvEncoderTest.cs
@@ -10,6 +10,7 @@ using JT1078.Protocol.Enums;
 using JT1078.Flv.H264;
 using JT1078.Flv.MessagePack;
 using JT1078.Flv.Metadata;
+using System.Diagnostics;
 
 namespace JT1078.Flv.Test
 {
@@ -35,15 +36,12 @@ namespace JT1078.Flv.Test
 
             FlvEncoder encoder = new FlvEncoder();
             var contents = encoder.CreateFlvFrame(nalus);
-            byte[] tmp = new byte[FlvEncoder.VideoFlvHeaderBuffer.Length+ contents.Length];
-            Array.Copy(FlvEncoder.VideoFlvHeaderBuffer, tmp, FlvEncoder.VideoFlvHeaderBuffer.Length);
-            Array.Copy(contents,0,tmp, FlvEncoder.VideoFlvHeaderBuffer.Length, contents.Length);
             var filepath = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "H264", "JT1078_1.flv");
             if (File.Exists(filepath))
             {
                 File.Delete(filepath);
             }
-            File.WriteAllBytes(filepath, tmp);
+            File.WriteAllBytes(filepath, contents);
         }
 
         [Fact]
@@ -79,7 +77,6 @@ namespace JT1078.Flv.Test
                     File.Delete(filepath);
                 }
                 fileStream = new FileStream(filepath, FileMode.OpenOrCreate, FileAccess.Write);
-                fileStream.Write(FlvEncoder.VideoFlvHeaderBuffer);
                 var totalPage = (h264NALULs.Count + 10 - 1) / 10;
                 for(var i=0;i< totalPage; i++)
                 {
@@ -136,7 +133,6 @@ namespace JT1078.Flv.Test
                 }
 
                 fileStream = new FileStream(filepath, FileMode.OpenOrCreate, FileAccess.Write);
-                fileStream.Write(FlvEncoder.VideoFlvHeaderBuffer);
                 var totalPage = (h264NALULs.Count + 10 - 1) / 10;
                 for (var i = 0; i < totalPage; i++)
                 {
@@ -192,9 +188,7 @@ namespace JT1078.Flv.Test
                     }
                 }
 
-                fileStream = new FileStream(filepath, FileMode.OpenOrCreate, FileAccess.Write);
-                fileStream.Write(FlvEncoder.VideoFlvHeaderBuffer);
-                
+                fileStream = new FileStream(filepath, FileMode.OpenOrCreate, FileAccess.Write);                
                 var totalPage = (h264NALULs.Count + 10 - 1) / 10;
                 for (var i = 0; i < totalPage; i++)
                 {
@@ -251,17 +245,16 @@ namespace JT1078.Flv.Test
                 }
                 //var tmp1 = h264NALULs.Where(w => w.NALUHeader.NalUnitType == 7).ToList();
                 List<SPSInfo> tmpSpss = new List<SPSInfo>();
-                List<ushort> times = new List<ushort>();
+                List<ulong> times = new List<ulong>();
                 List<int> type = new List<int>();
-                //foreach (var item in h264NALULs)
-                //{
-                //    //ExpGolombReader expGolombReader = new ExpGolombReader(item.RawData);
-                //    type.Add(item.NALUHeader.NalUnitType);
-                //    times.Add(item.LastFrameInterval);
-                //    //tmpSpss.Add(expGolombReader.ReadSPS());
-                //}
+                foreach (var item in h264NALULs)
+                {
+                    //ExpGolombReader expGolombReader = new ExpGolombReader(item.RawData);
+                    //type.Add(item.NALUHeader.NalUnitType);
+                    times.Add(item.Timestamp);
+                    //tmpSpss.Add(expGolombReader.ReadSPS());
+                }
                 fileStream = new FileStream(filepath, FileMode.OpenOrCreate, FileAccess.Write);
-                fileStream.Write(FlvEncoder.VideoFlvHeaderBuffer);
                 var totalPage = (h264NALULs.Count + 10 - 1) / 10;
                 for (var i = 0; i < totalPage; i++)
                 {
diff --git a/src/JT1078.Flv.Test/H264/index.html b/src/JT1078.Flv.Test/H264/index.html
index 46fb3eb..0a06bc3 100644
--- a/src/JT1078.Flv.Test/H264/index.html
+++ b/src/JT1078.Flv.Test/H264/index.html
@@ -9,13 +9,20 @@
 <body>
     <video muted="muted" webkit-playsinline="true" autoplay="true" id="player"></video>
     <script>
-
+        //ref https://github.com/bilibili/flv.js/issues/427
+        //ref https://github.com/bilibili/flv.js/issues/498
         if (flvjs.isSupported()) {
             var player = document.getElementById('player');
             var flvPlayer = flvjs.createPlayer({
                 type: 'flv',
-                url: "ws://localhost:1818/jt1078live?token=" + Math.floor((Math.random() * 1000000) + 1)
-                //url: "jt1078_5.flv"
+                lazyLoad: false,
+                lazyLoadMaxDuration: 0,
+                lazyLoadRecoverDuration: 0,
+                deferLoadAfterSourceOpen: false,
+                fixAudioTimestampGap: false,
+                //url: "ws://localhost:1818/jt1078live?token=" + Math.floor((Math.random() * 1000000) + 1)
+                //isLive: true,
+                url: "JT1078_3.flv"
             });
             var width, height, flag;
             flvPlayer.attachMediaElement(player);
diff --git a/src/JT1078.Flv.Test/JT1078.Flv.Test.csproj b/src/JT1078.Flv.Test/JT1078.Flv.Test.csproj
index fe9724d..6afd203 100644
--- a/src/JT1078.Flv.Test/JT1078.Flv.Test.csproj
+++ b/src/JT1078.Flv.Test/JT1078.Flv.Test.csproj
@@ -40,6 +40,9 @@
     <None Update="H264\JT1078_4.txt">
       <CopyToOutputDirectory>Always</CopyToOutputDirectory>
     </None>
+    <None Update="H264\JT1078_5.txt">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </None>
   </ItemGroup>
 
 </Project>
diff --git a/src/JT1078.Flv/FlvEncoder.cs b/src/JT1078.Flv/FlvEncoder.cs
index c5b46e2..d8def8d 100644
--- a/src/JT1078.Flv/FlvEncoder.cs
+++ b/src/JT1078.Flv/FlvEncoder.cs
@@ -22,6 +22,9 @@ namespace JT1078.Flv
             public uint PreviousTagSize { get; set;}
             public uint LastFrameInterval { get; set; }
             public uint LastIFrameInterval { get; set; }
+            public ulong Timestamp { get; set; }
+            public uint Interval { get; set; }
+            public JT1078DataType DataType { get; set; }
         }
 
         public static readonly byte[] VideoFlvHeaderBuffer;
@@ -122,7 +125,7 @@ namespace JT1078.Flv
                 FlvArrayPool.Return(buffer);
             }
         }
-        public byte[] CreateVideoTagOtherFrame(FlvFrameInfo flvFrameInfo, H264NALU nALU)
+        public byte[] CreateVideoTagOtherFrame(FlvFrameInfo flvFrameInfo, H264NALU nALU, H264NALU sei)
         {
             byte[] buffer = FlvArrayPool.Rent(65535);
             try
@@ -137,21 +140,30 @@ namespace JT1078.Flv
                 //flv body tag body
                 flvTags.VideoTagsData = new VideoTags();
                 flvTags.VideoTagsData.VideoData = new AvcVideoPacke();
-                flvTags.VideoTagsData.VideoData.CompositionTime = 0;
                 flvTags.VideoTagsData.VideoData.AvcPacketType = AvcPacketType.Raw;
                 if (nALU.NALUHeader.NalUnitType == 5 || nALU.DataType == JT1078DataType.视频I帧)
                 {
                     flvTags.VideoTagsData.FrameType = FrameType.KeyFrame;
-                    flvTags.Timestamp = flvFrameInfo.LastIFrameInterval;
                 }
                 else
                 {
-                    flvTags.Timestamp = flvFrameInfo.LastFrameInterval;
                     flvTags.VideoTagsData.FrameType = FrameType.InterFrame;
                 }
-#warning Timestamp时间戳没有控制好
-                flvTags.Timestamp = flvFrameInfo.LastIFrameInterval;
-                flvTags.VideoTagsData.VideoData.Data = nALU.RawData;
+                if(flvFrameInfo.DataType== JT1078DataType.视频I帧)
+                {
+                    flvTags.VideoTagsData.VideoData.CompositionTime = flvFrameInfo.LastIFrameInterval;
+                }
+                else
+                {
+                    flvTags.VideoTagsData.VideoData.CompositionTime = flvFrameInfo.LastFrameInterval;
+                }
+                flvTags.Timestamp = flvFrameInfo.Interval;
+                flvTags.VideoTagsData.VideoData.MultiData = new List<byte[]>();
+                flvTags.VideoTagsData.VideoData.MultiData.Add(nALU.RawData);
+                if(sei!=null && sei.RawData!=null && sei.RawData.Length > 0)
+                {
+                    flvTags.VideoTagsData.VideoData.MultiData.Add(sei.RawData);
+                }
                 flvMessagePackWriter.WriteFlvTag(flvTags);
                 return flvMessagePackWriter.FlushAndGetArray();
             }
@@ -160,14 +172,13 @@ namespace JT1078.Flv
                 FlvArrayPool.Return(buffer);
             }
         }
-
         public byte[] CreateFlvFrame(List<H264NALU> nALUs,int minimumLength = 65535)
         {
             byte[] buffer = FlvArrayPool.Rent(minimumLength);
             try
             {
                 FlvMessagePackWriter flvMessagePackWriter = new FlvMessagePackWriter(buffer);
-                H264NALU sps=null, pps=null;
+                H264NALU sps=null, pps=null, sei=null;
                 SPSInfo spsInfo = new SPSInfo();
                 foreach (var naln in nALUs)
                 {
@@ -177,19 +188,25 @@ namespace JT1078.Flv
                         if (VideoSPSDict.TryGetValue(key, out var spsInfoCache))
                         {
                             //todo: 主次码流
-                            if (spsInfoCache.height != spsInfo.height && spsInfoCache.width != spsInfo.width)
-                            {
-                                if (FlvFrameInfoDict.TryGetValue(key, out FlvFrameInfo flvFrameInfo))
-                                {
-                                    CreateFlvKeyFrame(ref flvMessagePackWriter, key, sps.RawData, pps.RawData, spsInfo);
-                                    VideoSPSDict.TryUpdate(key, spsInfo, spsInfo);
-                                    flvFrameInfo.LastIFrameInterval = 0;
-                                    FlvFrameInfoDict.TryUpdate(key, flvFrameInfo, flvFrameInfo);
-                                }
-                            }
+                            //if (spsInfoCache.height != spsInfo.height && spsInfoCache.width != spsInfo.width)
+                            //{
+                            //    if (FlvFrameInfoDict.TryGetValue(key, out FlvFrameInfo flvFrameInfo))
+                            //    {
+                            //        var rawData = H264Decoder.DiscardEmulationPreventionBytes(sps.RawData);
+                            //        ExpGolombReader h264GolombReader = new ExpGolombReader(rawData);
+                            //        spsInfo = h264GolombReader.ReadSPS();
+                            //        CreateFlvKeyFrame(ref flvMessagePackWriter, key, sps.RawData, pps.RawData, spsInfo);
+                            //        VideoSPSDict.TryUpdate(key, spsInfo, spsInfo);
+                            //        flvFrameInfo.LastIFrameInterval = 0;
+                            //        FlvFrameInfoDict.TryUpdate(key, flvFrameInfo, flvFrameInfo);
+                            //    }
+                            //}
                         }
                         else
                         {
+                            var rawData = H264Decoder.DiscardEmulationPreventionBytes(sps.RawData);
+                            ExpGolombReader h264GolombReader = new ExpGolombReader(rawData);
+                            spsInfo = h264GolombReader.ReadSPS();
                             flvMessagePackWriter.WriteArray(VideoFlvHeaderBuffer);
                             CreateFlvKeyFrame(ref flvMessagePackWriter, key, sps.RawData, pps.RawData, spsInfo);
                             VideoSPSDict.TryAdd(key, spsInfo);
@@ -205,27 +222,29 @@ namespace JT1078.Flv
                         case 1:// I/P/B
                             if (FlvFrameInfoDict.TryGetValue(key, out FlvFrameInfo  flvFrameInfo))
                             {
-                                flvFrameInfo.LastIFrameInterval += naln.LastIFrameInterval;
-                                flvFrameInfo.LastFrameInterval += naln.LastFrameInterval;
+                                flvFrameInfo.LastIFrameInterval = naln.LastIFrameInterval;
+                                flvFrameInfo.LastFrameInterval  = naln.LastFrameInterval;
+                                uint interval = (uint)(naln.Timestamp - flvFrameInfo.Timestamp);
+                                flvFrameInfo.Interval += interval;
+                                flvFrameInfo.Timestamp = naln.Timestamp;
                                 // PreviousTagSize
                                 flvMessagePackWriter.WriteUInt32(flvFrameInfo.PreviousTagSize);
                                 // Data Tag Frame
-                                var flvFrameBuffer = CreateVideoTagOtherFrame(flvFrameInfo, naln);
+                                var flvFrameBuffer = CreateVideoTagOtherFrame(flvFrameInfo, naln, sei);
                                 flvMessagePackWriter.WriteArray(flvFrameBuffer);
                                 flvFrameInfo.PreviousTagSize = (uint)flvFrameBuffer.Length;
+                                flvFrameInfo.DataType = naln.DataType;
                                 FlvFrameInfoDict.TryUpdate(key, flvFrameInfo, flvFrameInfo);
                             }
                             break;
-                        case 7:// sps
+                        case 7:// SPS
                             sps = naln;
-                            var rawData = H264Decoder.DiscardEmulationPreventionBytes(naln.RawData);
-                            ExpGolombReader h264GolombReader = new ExpGolombReader(rawData);
-                            spsInfo = h264GolombReader.ReadSPS();    
                             break;
-                        case 8:// pps
+                        case 8:// PPS
                             pps = naln;
                             break;
                         case 6://SEI
+                            sei = naln;
                             break;
                         default:
                             break;
@@ -247,7 +266,7 @@ namespace JT1078.Flv
             flvMessagePackWriter.WriteArray(scriptTagFrameBuffer);
             //flv script tag PreviousTagSize
             flvMessagePackWriter.WriteUInt32((uint)scriptTagFrameBuffer.Length);
-            //flv body video tag
+            //flv body video tag 0
             var videoTagFrame0Buffer= CreateVideoTag0Frame(spsRawData, ppsRawData, spsInfo);
             flvMessagePackWriter.WriteArray(videoTagFrame0Buffer);
             uint videoTag0PreviousTagSize = (uint)videoTagFrame0Buffer.Length;
diff --git a/src/JT1078.Flv/H264/H264Decoder.cs b/src/JT1078.Flv/H264/H264Decoder.cs
index 30c1ee9..036287a 100644
--- a/src/JT1078.Flv/H264/H264Decoder.cs
+++ b/src/JT1078.Flv/H264/H264Decoder.cs
@@ -75,6 +75,7 @@ namespace JT1078.Flv.H264
             nALU.LogicChannelNumber = package.LogicChannelNumber;
             nALU.LastFrameInterval = package.LastFrameInterval;
             nALU.LastIFrameInterval = package.LastIFrameInterval;
+            nALU.Timestamp = package.Timestamp;
             if (startCodePrefix == 3)
             {
                 nALU.StartCodePrefix = H264NALU.Start1;
diff --git a/src/JT1078.Flv/H264/H264NALU.cs b/src/JT1078.Flv/H264/H264NALU.cs
index f187865..b366a4a 100644
--- a/src/JT1078.Flv/H264/H264NALU.cs
+++ b/src/JT1078.Flv/H264/H264NALU.cs
@@ -35,6 +35,12 @@ namespace JT1078.Flv.H264
         /// </summary>
         public ushort LastFrameInterval { get; set; }
         /// <summary>
+        /// 时间戳
+        /// 标识此RTP数据包当前帧的相对时间，单位毫秒（ms）。
+        /// 当数据类型为01000时，则没有该字段
+        /// </summary>
+        public ulong Timestamp { get; set; }
+        /// <summary>
         /// 数据体
         /// </summary>
         public byte[] RawData { get; set; }
diff --git a/src/JT1078.Flv/MessagePack/FlvMessagePackWriter_Flv.cs b/src/JT1078.Flv/MessagePack/FlvMessagePackWriter_Flv.cs
index 39f5552..8fdbc9d 100644
--- a/src/JT1078.Flv/MessagePack/FlvMessagePackWriter_Flv.cs
+++ b/src/JT1078.Flv/MessagePack/FlvMessagePackWriter_Flv.cs
@@ -65,7 +65,6 @@ namespace JT1078.Flv.MessagePack
         public void WriteVideoTags(VideoTags videoTags)
         {
             WriteByte((byte)((byte)videoTags.FrameType | (byte)videoTags.CodecId));
-#warning 只处理H.264媒体数据
             if (videoTags.CodecId== CodecId.AvcVideoPacke)
             {
                 WriteAvcVideoPacke(videoTags.VideoData);
@@ -85,10 +84,13 @@ namespace JT1078.Flv.MessagePack
             else if(videoPacke.AvcPacketType == AvcPacketType.Raw)
             {
                 WriteUInt24(videoPacke.CompositionTime);
-                if (videoPacke.Data != null && videoPacke.Data.Length>0)
+                if (videoPacke.MultiData != null && videoPacke.MultiData.Count>0)
                 {
-                    WriteInt32(videoPacke.Data.Length);
-                    WriteArray(videoPacke.Data);
+                    foreach(var item in videoPacke.MultiData)
+                    {
+                        WriteInt32(item.Length);
+                        WriteArray(item);
+                    }
                 }
             }
             else
diff --git a/src/JT1078.Flv/Metadata/AvcVideoPacke.cs b/src/JT1078.Flv/Metadata/AvcVideoPacke.cs
index b93a14c..9ac4311 100644
--- a/src/JT1078.Flv/Metadata/AvcVideoPacke.cs
+++ b/src/JT1078.Flv/Metadata/AvcVideoPacke.cs
@@ -12,7 +12,6 @@ namespace JT1078.Flv.Metadata
         public uint CompositionTime { get; set; }
 
         public AVCDecoderConfigurationRecord AVCDecoderConfiguration { get; set; }
-
-        public byte[] Data { get; set; }
+        public List<byte[]> MultiData { get; set; }
     }
 }