完善fmp4的编码及发送

src/JT1078.Gateway.Tests/JT1078.Gateway.TestNormalHosting/Program.cs

@@ -41,6 +41,8 @@ namespace JT1078.Gateway.TestNormalHosting
                     services.AddSingleton<FlvEncoder>();
                     //hls视频解码器
                     services.AddSingleton<TSEncoder>();
+                    //h264
+                    services.AddSingleton<H264Decoder>();
                     services.AddSingleton<M3U8FileManage>();
                     //添加hls依赖项
                     services.AddHlsGateway(hostContext.Configuration);

src/JT1078.Gateway.Tests/JT1078.Gateway.TestNormalHosting/Services/JT1078FMp4NormalMsgHostedService.cs

@@ -26,19 +26,22 @@ namespace JT1078.Gateway.TestNormalHosting.Services
         private const string ikey = "IFMp4KEY";
         private MessageDispatchDataService messageDispatchDataService;
         private ConcurrentDictionary<string, List<H264NALU>> avFrameDict;
-
+        private H264Decoder H264Decoder;
         public JT1078FMp4NormalMsgHostedService(
             MessageDispatchDataService messageDispatchDataService,
             IMemoryCache memoryCache,
             ILoggerFactory loggerFactory,
             FMp4Encoder fM4Encoder,
+            H264Decoder h264Decoder,
             JT1078HttpSessionManager httpSessionManager)
         {
             Logger = loggerFactory.CreateLogger<JT1078FMp4NormalMsgHostedService>();
             HttpSessionManager = httpSessionManager;
             FM4Encoder = fM4Encoder;
+            H264Decoder= h264Decoder;
             this.memoryCache = memoryCache;
             this.messageDispatchDataService = messageDispatchDataService;
+            //todo:定时清理
             avFrameDict = new ConcurrentDictionary<string, List<H264NALU>>();
         }
         protected async override Task ExecuteAsync(CancellationToken stoppingToken)
@@ -46,7 +49,6 @@ namespace JT1078.Gateway.TestNormalHosting.Services
             while (!stoppingToken.IsCancellationRequested)
             {
                 var data = await messageDispatchDataService.FMp4Channel.Reader.ReadAsync();
-
                 try
                 {
                     if (Logger.IsEnabled(LogLevel.Debug))
@@ -54,25 +56,31 @@ namespace JT1078.Gateway.TestNormalHosting.Services
                         Logger.LogDebug(JsonSerializer.Serialize(HttpSessionManager.GetAll()));
                         Logger.LogDebug($"{data.SIM},{data.SN},{data.LogicChannelNumber},{data.Label3.DataType.ToString()},{data.Label3.SubpackageType.ToString()},{data.Bodies.ToHexString()}");
                     }
+                    var nalus = H264Decoder.ParseNALU(data);
                     string key = $"{data.GetKey()}_{ikey}";
                     if (data.Label3.DataType == Protocol.Enums.JT1078DataType.视频I帧)
                     {
-                        memoryCache.Set(key, data);
+                        var moovBuffer = FM4Encoder.EncoderMoovBox(
+                            nalus.FirstOrDefault(f => f.NALUHeader.NalUnitType == NalUnitType.SPS),
+                            nalus.FirstOrDefault(f => f.NALUHeader.NalUnitType == NalUnitType.PPS));
+                        memoryCache.Set(key, moovBuffer);
                     }
-                    var httpSessions = HttpSessionManager.GetAllBySimAndChannelNo(data.SIM.TrimStart('0'), data.LogicChannelNumber);
-                    var firstHttpSessions = httpSessions.Where(w => !w.FirstSend).ToList();
-                    if (firstHttpSessions.Count > 0)
+                    //查找第一帧为I帧，否则不推送
+                    if (memoryCache.TryGetValue(key, out byte[] moov))
                     {
-                        try
+                        var httpSessions = HttpSessionManager.GetAllBySimAndChannelNo(data.SIM.TrimStart('0'), data.LogicChannelNumber);
+                        var firstHttpSessions = httpSessions.Where(w => !w.FirstSend).ToList();
+                        if (firstHttpSessions.Count > 0)
                         {
-                            if (memoryCache.TryGetValue(key, out JT1078Package idata))
+                            try
                             {
                                 try
                                 {
+                                    var ftyp = FM4Encoder.EncoderFtypBox();
                                     foreach (var session in firstHttpSessions)
                                     {
-                                        var fmp4VideoBuffer = FM4Encoder.EncoderVideo(idata, session.FMp4EncoderInfo, true);
-                                        HttpSessionManager.SendAVData(session, fmp4VideoBuffer, true);             
+                                        HttpSessionManager.SendAVData(session, ftyp, true);
+                                        HttpSessionManager.SendAVData(session, moov, false);
                                     }
                                 }
                                 catch (Exception ex)
@@ -80,26 +88,52 @@ namespace JT1078.Gateway.TestNormalHosting.Services
                                     Logger.LogError(ex, $"{data.SIM},{true},{data.SN},{data.LogicChannelNumber},{data.Label3.DataType.ToString()},{data.Label3.SubpackageType.ToString()},{data.Bodies.ToHexString()}");
                                 }
                             }
-                        }
-                        catch (Exception ex)
-                        {
-                            Logger.LogError(ex, $"{data.SIM},{true},{data.SN},{data.LogicChannelNumber},{data.Label3.DataType.ToString()},{data.Label3.SubpackageType.ToString()},{data.Bodies.ToHexString()}");
-                        }
-                    }
-                    var otherHttpSessions = httpSessions.Where(w => w.FirstSend).ToList();
-                    if (otherHttpSessions.Count > 0)
-                    {
-                        try
-                        {
-                            foreach (var session in otherHttpSessions)
+                            catch (Exception ex)
                             {
-                                var fmp4VideoBuffer = FM4Encoder.EncoderVideo(data, session.FMp4EncoderInfo, false);
-                                HttpSessionManager.SendAVData(session, fmp4VideoBuffer, false);
+                                Logger.LogError(ex, $"{data.SIM},{true},{data.SN},{data.LogicChannelNumber},{data.Label3.DataType.ToString()},{data.Label3.SubpackageType.ToString()},{data.Bodies.ToHexString()}");
                             }
                         }
-                        catch (Exception ex)
+                        var otherHttpSessions = httpSessions.Where(w => w.FirstSend).ToList();
+                        if (otherHttpSessions.Count > 0)
                         {
-                            Logger.LogError(ex, $"{data.SIM},{false},{data.SN},{data.LogicChannelNumber},{data.Label3.DataType.ToString()},{data.Label3.SubpackageType.ToString()},{data.Bodies.ToHexString()}");
+                            try
+                            {
+                                var firstNALU = nalus.FirstOrDefault();
+                                if (firstNALU == null)
+                                {
+                                    continue;
+                                }
+                                if(!avFrameDict.TryGetValue(firstNALU.GetKey(),out List<H264NALU> cacheNALU))
+                                {
+                                    cacheNALU = new List<H264NALU>();
+                                    avFrameDict.TryAdd(firstNALU.GetKey(), cacheNALU);
+                                }
+                                foreach (var nalu in nalus)
+                                {
+                                    if (nalu.Slice)
+                                    {
+                                        //H264 NALU slice first_mb_in_slice
+                                        cacheNALU.Add(nalu);
+                                    }
+                                    else
+                                    {
+                                        if (cacheNALU.Count > 0)
+                                        {
+                                            foreach (var session in otherHttpSessions)
+                                            {
+                                                var fmp4VideoBuffer = FM4Encoder.EncoderOtherVideoBox(cacheNALU);
+                                                HttpSessionManager.SendAVData(session, fmp4VideoBuffer, false);
+                                            }
+                                            cacheNALU.Clear();
+                                        }
+                                        cacheNALU.Add(nalu);
+                                    }
+                                }
+                            }
+                            catch (Exception ex)
+                            {
+                                Logger.LogError(ex, $"{data.SIM},{false},{data.SN},{data.LogicChannelNumber},{data.Label3.DataType.ToString()},{data.Label3.SubpackageType.ToString()},{data.Bodies.ToHexString()}");
+                            }
                         }
                     }
                 }

src/JT1078.Gateway.Tests/JT1078.Gateway.TestNormalHosting/Services/MessageDispatchHostedService.cs

@@ -33,8 +33,8 @@ namespace JT1078.Gateway.TestNormalHosting.Services
                 var merge = JT1078.Protocol.JT1078Serializer.Merge(package);
                 if (merge != null)
                 {
-                    await messageDispatchDataService.HlsChannel.Writer.WriteAsync(merge, stoppingToken);
-                    await messageDispatchDataService.FlvChannel.Writer.WriteAsync(merge, stoppingToken);
+                    //await messageDispatchDataService.HlsChannel.Writer.WriteAsync(merge, stoppingToken);
+                    //await messageDispatchDataService.FlvChannel.Writer.WriteAsync(merge, stoppingToken);
                     await messageDispatchDataService.FMp4Channel.Writer.WriteAsync(merge, stoppingToken);
                 }
             });

src/JT1078.Gateway.Tests/JT1078.Gateway.TestNormalHosting/appsettings.json

@@ -13,7 +13,8 @@
     }
   },
   "JT1078Configuration": {
-    "HttpPort": 15555
+    "HttpPort": 15555,
+    "TcpPort": 10888
   },
   "M3U8Option": {
     "TsPathSimParamName": "sim",

src/JT1078.Gateway.Tests/JT1078.Gateway.TestNormalHosting/wwwroot/fmp4_demo/index.html

@@ -172,8 +172,9 @@
             ws.binaryType = 'arraybuffer';
             ws.onmessage = function (e) {
                 //当客户端收到服务端发来的消息时，触发onmessage事件，参数e.data包含server传递过来的数据
-                console.log(e.data);
-                putPacket(e.data);
+                //console.log(e.data);
+                //putPacket(e.data);
+                source_buffer.appendBuffer(e.data);
             }
         }
 

src/JT1078.Gateway/JT1078.Gateway.csproj

@@ -39,8 +39,8 @@
   </ItemGroup>
 
   <ItemGroup>
-    <PackageReference Include="JT1078.FMp4" Version="1.0.0-preview2" />
     <PackageReference Include="JT1078.Hls" Version="1.1.0-preview3" />
+    <PackageReference Include="JT1078.FMp4" Version="1.0.0-preview4" />
     <PackageReference Include="JT1078.Flv" Version="1.1.0" />
     <PackageReference Include="Microsoft.Extensions.Caching.Abstractions" Version="5.0.0" />
     <PackageReference Include="System.IO.Pipelines" Version="5.0.1" />

src/JT1078.Gateway/JT1078.Gateway.xml

@@ -195,11 +195,6 @@
             会话Id
             </summary>
         </member>
-        <member name="P:JT1078.Gateway.Metadata.JT1078HttpContext.FMp4EncoderInfo">
-            <summary>
-            FMp4编码信息
-            </summary>
-        </member>
         <member name="P:JT1078.Gateway.Metadata.JT1078HttpContext.Context">
             <summary>
             http上下文

src/JT1078.Gateway/Metadata/JT1078HttpContext.cs

@@ -19,10 +19,6 @@ namespace JT1078.Gateway.Metadata
         /// </summary>
         public string SessionId { get; }
         /// <summary>
-        /// FMp4编码信息
-        /// </summary>
-        public FMp4EncoderInfo FMp4EncoderInfo { get; set; } = new FMp4EncoderInfo();
-        /// <summary>
         /// http上下文
         /// </summary>
         [JsonIgnore]