1.升级net6

2.完善fmp4编码

.github/workflows/dotnetcore.yml

@@ -12,7 +12,7 @@ jobs:
     - name: Setup .NET Core
       uses: actions/setup-dotnet@master
       with:
-        dotnet-version: 5.0.100
+        dotnet-version: 6.0.100
     - name: dotnet info
       run: dotnet --info
     - name: dotnet restore

src/JT1078.AV.Benchmark/JT1078.AV.Benchmark.csproj

@@ -2,11 +2,11 @@
 
   <PropertyGroup>
     <OutputType>Exe</OutputType>
-    <TargetFramework>net5.0</TargetFramework>
+    <TargetFramework>net6.0</TargetFramework>
   </PropertyGroup>
   <ItemGroup>
-    <PackageReference Include="BenchmarkDotNet" Version="0.12.1" />
-    <PackageReference Include="BenchmarkDotNet.Diagnostics.Windows" Version="0.12.1" />
+    <PackageReference Include="BenchmarkDotNet" Version="0.13.1" />
+    <PackageReference Include="BenchmarkDotNet.Diagnostics.Windows" Version="0.13.1" />
     <PackageReference Include="System.Buffers" Version="4.5.1" />
     <PackageReference Include="System.Memory" Version="4.5.4" />
   </ItemGroup>

src/JT1078.AV.Benchmark/JT1078AVEncoderContext.cs

@@ -72,24 +72,6 @@ namespace JT1078.AV.Benchmark
                     break;
                 }
                 List<H264NALU> h264NALUs = h264Decoder.ParseNALU(package);
-                foreach (var nalu in h264NALUs)
-                {
-                    if (nalu.Slice)
-                    {
-                        //H264 NALU slice first_mb_in_slice
-                        nalus.Add(nalu);
-                    }
-                    else
-                    {
-                        if (nalus.Count > 0)
-                        {
-                            FMp4H264NALUs = new List<H264NALU>(nalus);
-                            segmentFlag = true;
-                            nalus.Clear();
-                        }
-                        nalus.Add(nalu);
-                    }
-                }
             }
         }
 
@@ -117,7 +99,8 @@ namespace JT1078.AV.Benchmark
         {
             for (var i = 0; i < N; i++)
             {
-                var buffer = fmp4Encoder.OtherVideoBox(FMp4H264NALUs);
+                //todo:OtherVideoBox
+                //var buffer = fmp4Encoder.OtherVideoBox(FMp4H264NALUs);
             }
         }
     }
@@ -126,7 +109,7 @@ namespace JT1078.AV.Benchmark
     {
         public JT1078AVEncoderConfig()
         {
-            AddJob(Job.Default.WithGcServer(false).WithToolchain(CsProjCoreToolchain.NetCoreApp50).WithPlatform(Platform.AnyCpu));
+            AddJob(Job.Default.WithGcServer(false).WithToolchain(CsProjCoreToolchain.NetCoreApp60).WithPlatform(Platform.AnyCpu));
         }
     }
 }

src/JT1078.FMp4.Test/JT1078.FMp4.Test.csproj

@@ -1,19 +1,19 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net5.0</TargetFramework>
+    <TargetFramework>net6.0</TargetFramework>
 
     <IsPackable>false</IsPackable>
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.8.0" />
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.0.0" />
     <PackageReference Include="xunit" Version="2.4.1" />
     <PackageReference Include="xunit.runner.visualstudio" Version="2.4.3">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
     </PackageReference>
-    <PackageReference Include="coverlet.collector" Version="1.3.0">
+    <PackageReference Include="coverlet.collector" Version="3.1.0">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
     </PackageReference>
@@ -39,9 +39,6 @@
     </None>   
     <None Update="FMP4\fragmented_demo_trun.txt">
       <CopyToOutputDirectory>Always</CopyToOutputDirectory>
-    </None>   
-    <None Update="H264\index.html">
-      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
     </None>
   </ItemGroup>
   

src/JT1078.FMp4.Test/JT1078ToFMp4Box_Test.cs

@@ -438,107 +438,11 @@ namespace JT1078.FMp4.Test
 
         [Fact]
         public void Test4()
-        {
-            FMp4Encoder fMp4Encoder = new FMp4Encoder();
-            H264Decoder h264Decoder = new H264Decoder();
-            var packages = ParseNALUTests();
-            var filepath = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "H264", "JT1078_5.mp4");
-            if (File.Exists(filepath))
-            {
-                File.Delete(filepath);
-            }
-
-            using var fileStream = new FileStream(filepath, FileMode.OpenOrCreate, FileAccess.Write);
-            var ftyp = fMp4Encoder.FtypBox();
-            fileStream.Write(ftyp);
-
-            var iNalus = h264Decoder.ParseNALU(packages[0]);
-            //判断第一帧是否关键帧
-            var moov = fMp4Encoder.VideoMoovBox(
-                iNalus.FirstOrDefault(f => f.NALUHeader.NalUnitType == NalUnitType.SPS),
-                iNalus.FirstOrDefault(f => f.NALUHeader.NalUnitType == NalUnitType.PPS));
-            fileStream.Write(moov);
-
-            List<H264NALU> nalus = new List<H264NALU>();
-            foreach (var package in packages)
-            {
-                List<H264NALU> h264NALUs = h264Decoder.ParseNALU(package);
-                foreach (var nalu in h264NALUs)
-                {
-                    if (nalu.Slice)
-                    {
-                        //H264 NALU slice first_mb_in_slice
-                        nalus.Add(nalu);
-                    }
-                    else
-                    {
-                        if (nalus.Count > 0)
-                        {
-                            var otherBuffer = fMp4Encoder.OtherVideoBox(nalus);
-                            fileStream.Write(otherBuffer);
-                            nalus.Clear();
-                        }
-                        nalus.Add(nalu);
-                    }
-                }
-            }
-            fileStream.Close();
-        }
-
-        [Fact]
-        public void Test5()
-        {
-            FMp4Encoder fMp4Encoder = new FMp4Encoder();
-            H264Decoder h264Decoder = new H264Decoder();
-            var packages = ParseNALUTests();
-            var filepath = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "H264", "JT1078_6.mp4");
-            if (File.Exists(filepath))
-            {
-                File.Delete(filepath);
-            }
-            using var fileStream = new FileStream(filepath, FileMode.OpenOrCreate, FileAccess.Write);
-
-            var ftyp = fMp4Encoder.FtypBox();
-            fileStream.Write(ftyp);
-
-            var iNalus = h264Decoder.ParseNALU(packages[0]);
-            //判断第一帧是否关键帧
-            var moov = fMp4Encoder.VideoMoovBox(
-                iNalus.FirstOrDefault(f => f.NALUHeader.NalUnitType == NalUnitType.SPS),
-                iNalus.FirstOrDefault(f => f.NALUHeader.NalUnitType == NalUnitType.PPS));
-            fileStream.Write(moov);
-
-            List<H264NALU> nalus = new List<H264NALU>();
-            foreach (var package in packages)
-            {
-                List<H264NALU> h264NALUs = h264Decoder.ParseNALU(package);
-                if (package.Label3.DataType == Protocol.Enums.JT1078DataType.视频I帧)
-                {
-                    if (nalus.Count > 0)
-                    {
-                        var otherBuffer = fMp4Encoder.OtherVideoBox(nalus);
-                        fileStream.Write(otherBuffer);
-                        nalus.Clear();
-                    }
-                }
-                nalus = nalus.Concat(h264NALUs).ToList();
-            }
-            if (nalus.Count > 0)
-            {
-                var otherBuffer = fMp4Encoder.OtherVideoBox(nalus);
-                fileStream.Write(otherBuffer);
-                nalus.Clear();
-            }
-            fileStream.Close();
-        }
-
-        [Fact]
-        public void Test6()
         {
             FMp4Encoder fMp4Encoder = new FMp4Encoder();
             H264Decoder h264Decoder = new H264Decoder();
             var packages = ParseNALUTests1();
-            var filepath = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "H264", "JT1078_7.mp4");
+            var filepath = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "H264", "JT1078_7_3.mp4");
             if (File.Exists(filepath))
             {
                 File.Delete(filepath);
@@ -551,38 +455,30 @@ namespace JT1078.FMp4.Test
             var iPackage = packages.FirstOrDefault(f => f.Label3.DataType == JT1078DataType.视频I帧);
             var iNalus = h264Decoder.ParseNALU(iPackage);
             //判断第一帧是否关键帧
-            var moov = fMp4Encoder.VideoMoovBox(
+            var moov = fMp4Encoder.MoovBox(
                 iNalus.FirstOrDefault(f => f.NALUHeader.NalUnitType == NalUnitType.SPS),
                 iNalus.FirstOrDefault(f => f.NALUHeader.NalUnitType == NalUnitType.PPS));
             fileStream.Write(moov);
-
-            List<H264NALU> nalus = new List<H264NALU>();
+            List<JT1078Package> tmp = new List<JT1078Package>();
             foreach (var package in packages)
             {
-                List<H264NALU> h264NALUs = h264Decoder.ParseNALU(package);
                 if (package.Label3.DataType == Protocol.Enums.JT1078DataType.视频I帧)
                 {
-                    if (nalus.Count > 0)
+                    if (tmp.Count>0)
                     {
                         fileStream.Write(fMp4Encoder.StypBox());
-                        var otherBuffer = fMp4Encoder.OtherVideoBox(nalus);
+                        var otherBuffer = fMp4Encoder.OtherVideoBox(tmp);
                         fileStream.Write(otherBuffer);
-                        nalus.Clear();
+                        tmp.Clear();
                     }
                 }
-                nalus = nalus.Concat(h264NALUs).ToList();
-            }
-            if (nalus.Count > 0)
-            {
-                var otherBuffer = fMp4Encoder.OtherVideoBox(nalus);
-                fileStream.Write(otherBuffer);
-                nalus.Clear();
+                tmp.Add(package);
             }
             fileStream.Close();
         }
 
         [Fact]
-        public void Test6_2()
+        public void Test4_2()
         {
             var filepath = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "H264", "JT1078_7.h264");
             if (File.Exists(filepath))

src/JT1078.FMp4/Boxs/TrackFragmentHeaderBox.cs

@@ -49,7 +49,7 @@ namespace JT1078.FMp4
         public uint DefaultSampleSize { get; set; }
         /// <summary>
         /// TFHD_FLAG_DEFAULT_FLAGS
-        /// MOV_AUDIO == handler_type ? 0x02000000 : (0x00010000| 0x01000000);
+        /// MOV_AUDIO == handler_type ? TFHD_FLAG_AUDIO_TPYE : TFHD_FLAG_VIDEO_TPYE;
         /// </summary>
         public uint DefaultSampleFlags { get; set; }
         #endregion

src/JT1078.FMp4/FMp4Constants.cs

@@ -18,6 +18,25 @@ namespace JT1078.FMp4
         /// </summary>
         public static readonly DateTime UTCBaseTime = new DateTime(1904, 1, 1);
         /// <summary>
+        /// fmp4 FLAG_SEGMENT
+        /// </summary>
+        public const int FLAG_SEGMENT = 0x00000002;
+        /// <summary>
+        /// key frame
+        /// </summary>
+
+        public const int AV_FLAG_KEYFREAME = 0x0001;
+        /// <summary>
+        /// I frame
+        /// </summary>
+
+        public const int TREX_FLAG_SAMPLE_DEPENDS_ON_I_PICTURE = 0x02000000;
+        /// <summary>
+        /// p b frame
+        /// </summary>
+
+        public const int TREX_FLAG_SAMPLE_DEPENDS_ON_NOT_I_PICTURE = 0x01000000;
+        /// <summary>
         /// TKHD_FLAG_ENABLED
         /// </summary>
         public const int TKHD_FLAG_ENABLED = 0x000001;
@@ -38,6 +57,14 @@ namespace JT1078.FMp4
         /// </summary>
         public const int TFHD_FLAG_SAMPLE_DESCRIPTION_INDEX = 0x00000002;
         /// <summary>
+        /// TFHD_FLAG_AUDIO_TPYE
+        /// </summary>
+        public const int TFHD_FLAG_AUDIO_TPYE = 0x02000000;
+        /// <summary>
+        /// TFHD_FLAG_VIDEO_TPYE
+        /// </summary>
+        public const int TFHD_FLAG_VIDEO_TPYE = (0x00010000| 0x01000000);
+        /// <summary>
         /// TFHD_FLAG_SAMPLE_DUR
         /// </summary>
         public const int TFHD_FLAG_DEFAULT_DURATION = 0x00000008;

src/JT1078.FMp4/FMp4Encoder.cs

@@ -19,28 +19,36 @@ namespace JT1078.FMp4
     /// ftyp
     /// moov
     /// styp 1
+    /// sidx 1
     /// moof 1
     /// mdat 1
     /// ...
     /// styp n
+    /// sidx n
     /// moof n
     /// mdat n
-    /// mfra
     /// ref: https://www.w3.org/TR/mse-byte-stream-format-isobmff/#movie-fragment-relative-addressing
     /// </summary>
     public class FMp4Encoder
     {
         Dictionary<string, TrackInfo> TrackInfos;
 
-        const uint DefaultSampleDuration = 40u;
-        const uint DefaultSampleFlags = 0x1010000;
-        const uint FirstSampleFlags = 33554432;
-        const uint TfhdFlags = 0x2003a;
-        //const uint TrunFlags = 0x205;
-        const uint TrunFlags = 0x205;
+        const uint TfhdFlags = FMp4Constants.TFHD_FLAG_DEFAULT_BASE_IS_MOOF |
+                               FMp4Constants.TFHD_FLAG_DEFAULT_SIZE|
+                               FMp4Constants.TFHD_FLAG_SAMPLE_DESCRIPTION_INDEX|
+                               FMp4Constants.TFHD_FLAG_DEFAULT_FLAGS;
+
+        const uint TrunFlags = FMp4Constants.TRUN_FLAG_DATA_OFFSET_PRESENT|
+                               FMp4Constants.TRUN_FLAG_FIRST_SAMPLE_FLAGS_PRESENT|
+                               FMp4Constants.TRUN_FLAG_SAMPLE_DURATION_PRESENT|
+                               FMp4Constants.TRUN_FLAG_SAMPLE_SIZE_PRESENT;
+
         const uint SampleDescriptionIndex = 1;
+
         const uint TrackID = 1;
 
+        H264Decoder h264Decoder = new H264Decoder();
+
         /// <summary>
         /// 
         /// </summary>
@@ -61,13 +69,15 @@ namespace JT1078.FMp4
             {
                 //ftyp
                 FileTypeBox fileTypeBox = new FileTypeBox();
-                fileTypeBox.MajorBrand = "isom";
-                fileTypeBox.MinorVersion = "\0\0\u0002\0";
+                fileTypeBox.MajorBrand = "msdh";
+                fileTypeBox.MinorVersion = "\0\0\0\0";
                 fileTypeBox.CompatibleBrands.Add("isom");
-                fileTypeBox.CompatibleBrands.Add("iso2");
-                fileTypeBox.CompatibleBrands.Add("avc1");
-                fileTypeBox.CompatibleBrands.Add("mp41");
+                fileTypeBox.CompatibleBrands.Add("mp42");
+                fileTypeBox.CompatibleBrands.Add("msdh");
+                fileTypeBox.CompatibleBrands.Add("msix");
+                // default‐base is‐moof flag
                 fileTypeBox.CompatibleBrands.Add("iso5");
+                // styp
                 fileTypeBox.CompatibleBrands.Add("iso6");
                 fileTypeBox.ToBuffer(ref writer);
                 var data = writer.FlushAndGetArray();
@@ -83,7 +93,7 @@ namespace JT1078.FMp4
         /// 编码moov盒子
         /// </summary>
         /// <returns></returns>
-        public byte[] VideoMoovBox(in H264NALU sps, in H264NALU pps)
+        public byte[] MoovBox(in H264NALU sps, in H264NALU pps)
         {
             byte[] buffer = FMp4ArrayPool.Rent(sps.RawData.Length + pps.RawData.Length + 1024);
             FMp4MessagePackWriter writer = new FMp4MessagePackWriter(buffer);
@@ -112,7 +122,6 @@ namespace JT1078.FMp4
                 movieBox.TrackBox.MediaBox.MediaHeaderBox = new MediaHeaderBox();
                 movieBox.TrackBox.MediaBox.MediaHeaderBox.CreationTime = 0;
                 movieBox.TrackBox.MediaBox.MediaHeaderBox.ModificationTime = 0;
-                //movieBox.TrackBox.MediaBox.MediaHeaderBox.Timescale = 1200000;
                 movieBox.TrackBox.MediaBox.MediaHeaderBox.Timescale = 1000;
                 movieBox.TrackBox.MediaBox.MediaHeaderBox.Duration = 0;
                 movieBox.TrackBox.MediaBox.HandlerBox = new HandlerBox();
@@ -173,7 +182,7 @@ namespace JT1078.FMp4
             try
             {
                 SegmentTypeBox stypTypeBox = new SegmentTypeBox();
-                stypTypeBox.MajorBrand = "isom";
+                stypTypeBox.MajorBrand = "msdh";
                 stypTypeBox.MinorVersion = "\0\0\0\0";
                 stypTypeBox.CompatibleBrands.Add("isom");
                 stypTypeBox.CompatibleBrands.Add("mp42");
@@ -193,19 +202,22 @@ namespace JT1078.FMp4
 
         /// <summary>
         /// 编码其他视频数据盒子
+        /// 注意：固定I帧解析 
+        /// I P P P P I P P P P I P P P P
+        /// todo:50ms或者一个关键帧进行切片
+        /// todo:优化编码
         /// </summary>
         /// <returns></returns>
-        public byte[] OtherVideoBox(in List<H264NALU> nalus)
+        public byte[] OtherVideoBox(in List<JT1078Package> nalus)
         {
-            byte[] buffer = FMp4ArrayPool.Rent(nalus.Sum(s => s.RawData.Length + s.StartCodePrefix.Length) + 4096);
+            byte[] buffer = FMp4ArrayPool.Rent(nalus.Sum(s => s.Bodies.Length) + 4096);
             FMp4MessagePackWriter writer = new FMp4MessagePackWriter(buffer);
             try
             {
                 var truns = new List<TrackRunBox.TrackRunInfo>();
-                List<byte[]> rawdatas = new List<byte[]>();
-                uint iSize = 0;
-                ulong lastTimestamp = 0;
                 string key = string.Empty;
+                ulong timestamp = 0;
+                uint subsegmentDuration=0;
                 for (var i = 0; i < nalus.Count; i++)
                 {
                     var nalu = nalus[i];
@@ -213,46 +225,44 @@ namespace JT1078.FMp4
                     {
                         key = nalu.GetKey();
                     }
-                    rawdatas.Add(nalu.RawData);
-                    if (nalu.DataType == Protocol.Enums.JT1078DataType.视频I帧)
+                    uint duration = 0;
+                    if (timestamp>0)
                     {
-                        iSize += (uint)(nalu.RawData.Length + nalu.StartCodePrefix.Length);
-                    }
-                    else
-                    {
-                        if (iSize > 0)
-                        {
-                            truns.Add(new TrackRunBox.TrackRunInfo()
-                            {
-                                SampleDuration=40,
-                                SampleSize = iSize,
-                            });
-                            iSize = 0;
-                        }
-                        truns.Add(new TrackRunBox.TrackRunInfo()
-                        {
-                            SampleDuration = 40,
-                            SampleSize = (uint)(nalu.RawData.Length + nalu.StartCodePrefix.Length),
-                        });
+                        duration=(uint)(nalu.Timestamp-timestamp);
                     }
-                    if (i == (nalus.Count - 1))
+                    truns.Add(new TrackRunBox.TrackRunInfo()
                     {
-                        lastTimestamp = nalu.Timestamp;
-                    }
+                        SampleDuration = duration,
+                        SampleSize = (uint)(nalu.Bodies.Length),
+                    });
+                    subsegmentDuration+=duration;
+                    timestamp=nalu.Timestamp;
                 }
-                if (TrackInfos.TryGetValue(key, out TrackInfo trackInfo))
+                if (!TrackInfos.TryGetValue(key, out TrackInfo trackInfo))
                 {
-                    if (trackInfo.SN == uint.MaxValue)
-                    {
-                        trackInfo.SN = 1;
-                    }
-                    trackInfo.SN++;
+                    trackInfo = new TrackInfo { SN = 1, DTS = 0, SubsegmentDuration=0 };
+                    TrackInfos.Add(key, trackInfo);
                 }
-                else
+                if (trackInfo.SN == uint.MaxValue)
                 {
-                    trackInfo = new TrackInfo { SN = 1, DTS = 0 };
-                    TrackInfos.Add(key, trackInfo);
+                    trackInfo.SN = 1;
                 }
+                trackInfo.SN++;
+                SegmentIndexBox segmentIndexBox = new SegmentIndexBox(1);
+                segmentIndexBox.ReferenceID = 1;
+                segmentIndexBox.EarliestPresentationTime = trackInfo.SubsegmentDuration;
+                segmentIndexBox.SegmentIndexs = new List<SegmentIndexBox.SegmentIndex>()
+                {
+                     new SegmentIndexBox.SegmentIndex
+                     {
+                          SubsegmentDuration=subsegmentDuration
+                     }
+                };
+
+                segmentIndexBox.ToBuffer(ref writer);
+
+                var current1 = writer.GetCurrentPosition();
+                //moof
                 var movieFragmentBox = new MovieFragmentBox();
                 movieFragmentBox.MovieFragmentHeaderBox = new MovieFragmentHeaderBox();
                 movieFragmentBox.MovieFragmentHeaderBox.SequenceNumber = trackInfo.SN;
@@ -260,23 +270,39 @@ namespace JT1078.FMp4
                 movieFragmentBox.TrackFragmentBox.TrackFragmentHeaderBox = new TrackFragmentHeaderBox(TfhdFlags);
                 movieFragmentBox.TrackFragmentBox.TrackFragmentHeaderBox.TrackID = TrackID;
                 movieFragmentBox.TrackFragmentBox.TrackFragmentHeaderBox.SampleDescriptionIndex = SampleDescriptionIndex;
-                movieFragmentBox.TrackFragmentBox.TrackFragmentHeaderBox.DefaultSampleDuration = DefaultSampleDuration;
                 movieFragmentBox.TrackFragmentBox.TrackFragmentHeaderBox.DefaultSampleSize = truns[0].SampleSize;
-                movieFragmentBox.TrackFragmentBox.TrackFragmentHeaderBox.DefaultSampleFlags = DefaultSampleFlags;
+                movieFragmentBox.TrackFragmentBox.TrackFragmentHeaderBox.DefaultSampleFlags = FMp4Constants.TFHD_FLAG_VIDEO_TPYE;
                 movieFragmentBox.TrackFragmentBox.TrackFragmentBaseMediaDecodeTimeBox = new TrackFragmentBaseMediaDecodeTimeBox();
-                movieFragmentBox.TrackFragmentBox.TrackFragmentBaseMediaDecodeTimeBox.BaseMediaDecodeTime = trackInfo.DTS;
-                trackInfo.DTS += (ulong)(truns.Count * DefaultSampleDuration);
+                movieFragmentBox.TrackFragmentBox.TrackFragmentBaseMediaDecodeTimeBox.BaseMediaDecodeTime =  trackInfo.SubsegmentDuration;
+                trackInfo.SubsegmentDuration+=subsegmentDuration;
                 TrackInfos[key] = trackInfo;
                 //trun
-                movieFragmentBox.TrackFragmentBox.TrackRunBox = new TrackRunBox(flags: TrunFlags);
-                movieFragmentBox.TrackFragmentBox.TrackRunBox.FirstSampleFlags = FirstSampleFlags;
+                movieFragmentBox.TrackFragmentBox.TrackRunBox = new TrackRunBox(1, flags: TrunFlags);
+                movieFragmentBox.TrackFragmentBox.TrackRunBox.FirstSampleFlags = FMp4Constants.TREX_FLAG_SAMPLE_DEPENDS_ON_I_PICTURE;
                 movieFragmentBox.TrackFragmentBox.TrackRunBox.TrackRunInfos = truns;
                 movieFragmentBox.ToBuffer(ref writer);
-
+                //mdat
                 var mediaDataBox = new MediaDataBox();
-                mediaDataBox.Data = rawdatas;
+                mediaDataBox.Data=new List<byte[]>();
+                foreach(var nalu in nalus)
+                {
+                    List<H264NALU> h264NALUs = h264Decoder.ParseNALU(nalu);
+                    if (h264NALUs!=null)
+                    {
+                        foreach(var n in h264NALUs)
+                        {
+                            mediaDataBox.Data.Add(n.RawData);
+                        }
+                    }
+                }
                 mediaDataBox.ToBuffer(ref writer);
 
+                var current2 = writer.GetCurrentPosition();
+                foreach (var postion in segmentIndexBox.ReferencedSizePositions)
+                {
+                    writer.WriteUInt32Return((uint)(current2 - current1), postion);
+                }
+
                 var data = writer.FlushAndGetArray();
                 return data;
             }
@@ -290,6 +316,7 @@ namespace JT1078.FMp4
         {
             public uint SN { get; set; }
             public ulong DTS { get; set; }
+            public ulong SubsegmentDuration { get; set; }
         }
     }
 }

src/JT1078.FMp4/JT1078.FMp4.csproj

@@ -1,8 +1,8 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFrameworks>netstandard2.0;netstandard2.1;net5.0;</TargetFrameworks>
-    <LangVersion>8.0</LangVersion>
+	<TargetFramework>net6.0</TargetFramework>
+    <LangVersion>10.0</LangVersion>
     <Copyright>Copyright 2019.</Copyright>
     <Authors>SmallChi(Koike)</Authors>
     <PackageId>JT1078.FMp4</PackageId>

src/JT1078.FMp4/JT1078.FMp4.xml

@@ -983,7 +983,7 @@
         <member name="P:JT1078.FMp4.TrackFragmentHeaderBox.DefaultSampleFlags">
             <summary>
             TFHD_FLAG_DEFAULT_FLAGS
-            MOV_AUDIO == handler_type ? 0x02000000 : (0x00010000| 0x01000000);
+            MOV_AUDIO == handler_type ? TFHD_FLAG_AUDIO_TPYE : TFHD_FLAG_VIDEO_TPYE;
             </summary>
         </member>
         <member name="T:JT1078.FMp4.TrackFragmentRandomAccessBox">
@@ -1222,6 +1222,26 @@
             
             </summary>
         </member>
+        <member name="F:JT1078.FMp4.FMp4Constants.FLAG_SEGMENT">
+            <summary>
+            fmp4 FLAG_SEGMENT
+            </summary>
+        </member>
+        <member name="F:JT1078.FMp4.FMp4Constants.AV_FLAG_KEYFREAME">
+            <summary>
+            key frame
+            </summary>
+        </member>
+        <member name="F:JT1078.FMp4.FMp4Constants.TREX_FLAG_SAMPLE_DEPENDS_ON_I_PICTURE">
+            <summary>
+            I frame
+            </summary>
+        </member>
+        <member name="F:JT1078.FMp4.FMp4Constants.TREX_FLAG_SAMPLE_DEPENDS_ON_NOT_I_PICTURE">
+            <summary>
+            p b frame
+            </summary>
+        </member>
         <member name="F:JT1078.FMp4.FMp4Constants.TKHD_FLAG_ENABLED">
             <summary>
             TKHD_FLAG_ENABLED
@@ -1247,6 +1267,16 @@
             TFHD_FLAG_SAMPLE_DESC
             </summary>
         </member>
+        <member name="F:JT1078.FMp4.FMp4Constants.TFHD_FLAG_AUDIO_TPYE">
+            <summary>
+            TFHD_FLAG_AUDIO_TPYE
+            </summary>
+        </member>
+        <member name="F:JT1078.FMp4.FMp4Constants.TFHD_FLAG_VIDEO_TPYE">
+            <summary>
+            TFHD_FLAG_VIDEO_TPYE
+            </summary>
+        </member>
         <member name="F:JT1078.FMp4.FMp4Constants.TFHD_FLAG_DEFAULT_DURATION">
             <summary>
             TFHD_FLAG_SAMPLE_DUR
@@ -1310,13 +1340,14 @@
             ftyp
             moov
             styp 1
+            sidx 1
             moof 1
             mdat 1
             ...
             styp n
+            sidx n
             moof n
             mdat n
-            mfra
             ref: https://www.w3.org/TR/mse-byte-stream-format-isobmff/#movie-fragment-relative-addressing
             </summary>
         </member>
@@ -1331,7 +1362,7 @@
             </summary>
             <returns></returns>
         </member>
-        <member name="M:JT1078.FMp4.FMp4Encoder.VideoMoovBox(JT1078.Protocol.H264.H264NALU@,JT1078.Protocol.H264.H264NALU@)">
+        <member name="M:JT1078.FMp4.FMp4Encoder.MoovBox(JT1078.Protocol.H264.H264NALU@,JT1078.Protocol.H264.H264NALU@)">
             <summary>
             编码moov盒子
             </summary>
@@ -1343,9 +1374,13 @@
             </summary>
             <returns></returns>
         </member>
-        <member name="M:JT1078.FMp4.FMp4Encoder.OtherVideoBox(System.Collections.Generic.List{JT1078.Protocol.H264.H264NALU}@)">
+        <member name="M:JT1078.FMp4.FMp4Encoder.OtherVideoBox(System.Collections.Generic.List{JT1078.Protocol.JT1078Package}@)">
             <summary>
             编码其他视频数据盒子
+            注意：固定I帧解析 
+            I P P P P I P P P P I P P P P
+            todo:50ms或者一个关键帧进行切片
+            todo:优化编码
             </summary>
             <returns></returns>
         </member>

src/JT1078.Flv.Test/JT1078.Flv.Test.csproj

@@ -1,17 +1,17 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net5.0</TargetFramework>
+    <TargetFramework>net6.0</TargetFramework>
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.8.0" />
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.0.0" />
     <PackageReference Include="xunit" Version="2.4.1" />
     <PackageReference Include="xunit.runner.visualstudio" Version="2.4.3">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
     </PackageReference>
-    <PackageReference Include="coverlet.collector" Version="1.3.0">
+    <PackageReference Include="coverlet.collector" Version="3.1.0">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
     </PackageReference>

src/JT1078.Flv/JT1078.Flv.csproj

@@ -1,8 +1,8 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFrameworks>netstandard2.0;netstandard2.1;net5.0;</TargetFrameworks>
-    <LangVersion>8.0</LangVersion>
+	<TargetFramework>net6.0</TargetFramework>
+    <LangVersion>10.0</LangVersion>
     <Copyright>Copyright 2019.</Copyright>
     <Authors>SmallChi(Koike)</Authors>
     <PackageId>JT1078.Flv</PackageId>
@@ -35,6 +35,6 @@
     </None>
   </ItemGroup>
   <ItemGroup>
-    <PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="5.0.0" />
+    <PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="6.0.0" />
   </ItemGroup>
 </Project>

src/JT1078.Hls.Test/JT1078.Hls.Test.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net5.0</TargetFramework>
+    <TargetFramework>net6.0</TargetFramework>
 
     <IsPackable>false</IsPackable>
   </PropertyGroup>
@@ -15,14 +15,14 @@
   </ItemGroup>
 
   <ItemGroup>
-    <PackageReference Include="Microsoft.Extensions.ObjectPool" Version="5.0.0" />
-    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.8.0" />
+    <PackageReference Include="Microsoft.Extensions.ObjectPool" Version="6.0.0" />
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.0.0" />
     <PackageReference Include="xunit" Version="2.4.1" />
     <PackageReference Include="xunit.runner.visualstudio" Version="2.4.3">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
     </PackageReference>
-    <PackageReference Include="coverlet.collector" Version="1.3.0">
+    <PackageReference Include="coverlet.collector" Version="3.1.0">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
     </PackageReference>

src/JT1078.Hls/JT1078.Hls.csproj

@@ -1,8 +1,8 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFrameworks>netstandard2.0;netstandard2.1;net5.0;</TargetFrameworks>
-    <LangVersion>8.0</LangVersion>
+	<TargetFramework>net6.0</TargetFramework>
+    <LangVersion>10.0</LangVersion>
     <Copyright>Copyright 2019.</Copyright>
     <Authors>SmallChi(Koike)</Authors>
     <PackageId>JT1078.Hls</PackageId>
@@ -31,7 +31,7 @@
     </None>
   </ItemGroup>
   <ItemGroup>
-    <PackageReference Include="System.Text.Encoding.CodePages" Version="5.0.0" />
+    <PackageReference Include="System.Text.Encoding.CodePages" Version="6.0.0" />
   </ItemGroup>
   <ItemGroup>
     <ProjectReference Include="..\JT1078.Protocol\JT1078.Protocol.csproj" />

src/JT1078.Protocol.Benchmark/JT1078.Protocol.Benchmark.csproj

@@ -2,15 +2,15 @@
 
   <PropertyGroup>
     <OutputType>Exe</OutputType>
-    <TargetFramework>net5.0</TargetFramework>
+    <TargetFramework>net6.0</TargetFramework>
   </PropertyGroup>
 
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|AnyCPU'">
     <Optimize>true</Optimize>
   </PropertyGroup>
   <ItemGroup>
-    <PackageReference Include="BenchmarkDotNet" Version="0.12.1" />
-    <PackageReference Include="BenchmarkDotNet.Diagnostics.Windows" Version="0.12.1" />
+    <PackageReference Include="BenchmarkDotNet" Version="0.13.1" />
+    <PackageReference Include="BenchmarkDotNet.Diagnostics.Windows" Version="0.13.1" />
     <PackageReference Include="System.Buffers" Version="4.5.1" />
     <PackageReference Include="System.Memory" Version="4.5.4" />
   </ItemGroup>

src/JT1078.Protocol.Test/H264/H264DecoderTest.cs

@@ -135,7 +135,6 @@ namespace JT1078.Protocol.Test.H264
                     nALUs = nALUs.Concat(nalus).ToList();
                 }
             }
-            var a = nALUs.Count(c => !c.Slice);
         }
     }
 }

src/JT1078.Protocol.Test/JT1078.Protocol.Test.csproj

@@ -1,14 +1,14 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net5.0</TargetFramework>
+    <TargetFramework>net6.0</TargetFramework>
 
     <IsPackable>false</IsPackable>
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="JT808" Version="2.3.0" />
-    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.8.0" />
+    <PackageReference Include="JT808" Version="2.4.5" />
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.0.0" />
     <PackageReference Include="xunit" Version="2.4.1" />
     <PackageReference Include="xunit.runner.visualstudio" Version="2.4.3">
       <PrivateAssets>all</PrivateAssets>

src/JT1078.Protocol/H264/H264Decoder.cs

@@ -134,7 +134,6 @@ namespace JT1078.Protocol.H264
             nALU.LastFrameInterval = package.LastFrameInterval;
             nALU.LastIFrameInterval = package.LastIFrameInterval;
             nALU.Timestamp = package.Timestamp;
-            nALU.Slice = (nalu[1] & 0x80)== 0x80;
             nALU.RawData = nalu.ToArray();
             if (startCodePrefix == 3)
             {

src/JT1078.Protocol/H264/H264NALU.cs

@@ -40,12 +40,6 @@ namespace JT1078.Protocol.H264
         /// 当数据类型为01000时，则没有该字段
         /// </summary>
         public ulong Timestamp { get; set; }
-
-        /// <summary>
-        /// 是否切片 0x80
-        /// H264 NALU slice first_mb_in_slice
-        /// </summary>
-        public bool Slice { get; set; }
         /// <summary>
         /// 数据体
         /// </summary>

src/JT1078.Protocol/H264/NALUHeader.cs

@@ -11,16 +11,19 @@ namespace JT1078.Protocol.H264
             ForbiddenZeroBit = (value & 0x80) >> 7;
             NalRefIdc = (value & 0x60) >> 5;
             NalUnitType = (NalUnitType)(value & 0x1f);
+            KeyFrame=NalUnitType== NalUnitType.IDR;
         }
         public NALUHeader(ReadOnlySpan<byte> value)
         {
             ForbiddenZeroBit = (value[0] & 0x80) >> 7;
             NalRefIdc = (value[0] & 0x60) >> 5;
             NalUnitType = (NalUnitType)(value[0] & 0x1f);
+            KeyFrame=NalUnitType== NalUnitType.IDR;
         }
         public int ForbiddenZeroBit { get; set; }
         public int NalRefIdc { get; set; }
         public NalUnitType NalUnitType { get; set; }
+        public bool KeyFrame { get; set; }
     }
 
     public enum NalUnitType : int

src/JT1078.Protocol/JT1078.Protocol.csproj

@@ -1,8 +1,8 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFrameworks>netstandard2.0;netstandard2.1;net5.0;</TargetFrameworks>
-    <LangVersion>8.0</LangVersion>
+    <TargetFramework>net6.0</TargetFramework>
+    <LangVersion>10.0</LangVersion>
     <Copyright>Copyright 2019.</Copyright>
     <Authors>SmallChi(Koike)</Authors>
     <PackageId>JT1078</PackageId>
@@ -21,12 +21,6 @@
     <DocumentationFile>JT1078.Protocol.xml</DocumentationFile>
   </PropertyGroup>
 
-
-  <ItemGroup Condition=" '$(TargetFramework)' == 'netstandard2.0' ">
-    <PackageReference Include="System.Memory" Version="4.5.4" />
-  </ItemGroup>
-
-
   <ItemGroup>
     <Compile Remove="Audio\FaacEncoder.cs" />
   </ItemGroup>
@@ -37,8 +31,4 @@
       <PackagePath></PackagePath>
     </None>
   </ItemGroup>
-
-  <ItemGroup>
-    <PackageReference Include="System.Text.Json" Version="4.7.2" />
-  </ItemGroup>
 </Project>

src/JT1078.Protocol/JT1078.Protocol.xml

@@ -176,12 +176,6 @@
             当数据类型为01000时，则没有该字段
             </summary>
         </member>
-        <member name="P:JT1078.Protocol.H264.H264NALU.Slice">
-            <summary>
-            是否切片 0x80
-            H264 NALU slice first_mb_in_slice
-            </summary>
-        </member>
         <member name="P:JT1078.Protocol.H264.H264NALU.RawData">
             <summary>
             数据体

src/JT1078.SignalR.Test/JT1078.SignalR.Test.csproj

@@ -1,9 +1,14 @@
 <Project Sdk="Microsoft.NET.Sdk.Web">
 
   <PropertyGroup>
-    <TargetFramework>net5.0</TargetFramework>
+    <TargetFramework>net6.0</TargetFramework>
   </PropertyGroup>
 
+  <ItemGroup>
+    <Content Remove="wwwroot\index.html" />
+    <Content Remove="wwwroot\signalr.min.js" />
+  </ItemGroup>
+
   <ItemGroup>
     <ProjectReference Include="..\JT1078.FMp4\JT1078.FMp4.csproj" />
   </ItemGroup>
@@ -26,5 +31,11 @@
     <None Include="..\..\doc\video\jt1078_6.txt" Link="H264\jt1078_6.txt">
       <CopyToOutputDirectory>Always</CopyToOutputDirectory>
     </None>
+    <None Include="wwwroot\index.html">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </None>
+    <None Include="wwwroot\signalr.min.js">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </None>
   </ItemGroup>
 </Project>

src/JT1078.SignalR.Test/Properties/launchSettings.json

@@ -0,0 +1,12 @@
+{
+  "profiles": {
+    "JT1078.SignalR.Test": {
+      "commandName": "Project",
+      "launchBrowser": true,
+      "environmentVariables": {
+        "ASPNETCORE_ENVIRONMENT": "Development"
+      },
+      "applicationUrl": "http://localhost:5000/"
+    }
+  }
+}

src/JT1078.SignalR.Test/Services/ToWebSocketService.cs

@@ -52,20 +52,11 @@ namespace JT1078.SignalR.Test.Services
         public void a()
         {
             List<JT1078Package> packages = new List<JT1078Package>();
-            //var lines = File.ReadAllLines(Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "H264", "jt1078_3.txt"));
-            //var lines = File.ReadAllLines(Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "H264", "jt1078_5.txt"));
             var lines = File.ReadAllLines(Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "H264", "jt1078_6.txt"));
             int mergeBodyLength = 0;
             foreach (var line in lines)
             {
-                //var data = line.Split(',');
-                //jt1078_5
-                //var bytes = data[1].ToHexBytes();
-                //jt1078_3
-                //var bytes = data[6].ToHexBytes();
-                //jt1078_6
                 var bytes = line.ToHexBytes();
-
                 JT1078Package package = JT1078Serializer.Deserialize(bytes);
                 mergeBodyLength += package.DataBodyLength;
                 var packageMerge = JT1078Serializer.Merge(package);
@@ -75,61 +66,31 @@ namespace JT1078.SignalR.Test.Services
                 }
             }
             List<byte[]> first = new List<byte[]>();
-            //var styp = fMp4Encoder.EncoderStypBox();
-            //first.Add(styp);
-            //q.Enqueue(styp);
             var ftyp = fMp4Encoder.FtypBox();
-            //q.Enqueue(ftyp);
             first.Add(ftyp);
             var package1 = packages[0];
             var nalus1 = h264Decoder.ParseNALU(package1);
-            var moov = fMp4Encoder.VideoMoovBox(
+            var moov = fMp4Encoder.MoovBox(
               nalus1.FirstOrDefault(f => f.NALUHeader.NalUnitType == NalUnitType.SPS),
               nalus1.FirstOrDefault(f => f.NALUHeader.NalUnitType == NalUnitType.PPS));
-            //q.Enqueue(moov);
             first.Add(moov);
             q.Add(first.SelectMany(s=>s).ToArray());
-            List<NalUnitType> filter = new List<NalUnitType>() { NalUnitType.SEI,NalUnitType.SPS,NalUnitType.PPS,NalUnitType.AUD};
-            List<H264NALU> nalus = new List<H264NALU>();
+            List<JT1078Package> tmp = new List<JT1078Package>();
+            //缓存组包到下一个I帧
             foreach (var package in packages)
             {
-                List<H264NALU> h264NALUs = h264Decoder.ParseNALU(package);
-                //if(package.Label3.DataType== Protocol.Enums.JT1078DataType.视频I帧)
-                //{
-                //    if (nalus.Count > 0)
-                //    {
-                //        var otherBuffer = fMp4Encoder.OtherVideoBox(nalus);
-                //        q.Add(fMp4Encoder.StypBox().Concat(otherBuffer).ToArray());
-                //        nalus.Clear();
-                //    }
-                //    else
-                //    {
-                //        nalus = nalus.Concat(h264NALUs).ToList();
-                //    }
-                //}
-                //else
-                //{
-                //    nalus = nalus.Concat(h264NALUs).ToList();
-                //}
-                foreach (var nalu in h264NALUs)
+                if (package.Label3.DataType == Protocol.Enums.JT1078DataType.视频I帧)
                 {
-                    if (nalu.Slice)
-                    {
-                        //H264 NALU slice first_mb_in_slice
-                        nalus.Add(nalu);
-                    }
-                    else
+                    if (tmp.Count>0)
                     {
-                        if (nalus.Count > 0)
-                        {
-                            q.Add(fMp4Encoder.StypBox());
-                            var otherBuffer = fMp4Encoder.OtherVideoBox(nalus);
-                            q.Add(otherBuffer);
-                            nalus.Clear();
-                        }
-                        nalus.Add(nalu);
+                        List<byte[]> buffer = new List<byte[]>();
+                        buffer.Add(fMp4Encoder.StypBox());
+                        buffer.Add(fMp4Encoder.OtherVideoBox(tmp));
+                        q.Add(buffer.SelectMany(s => s).ToArray());
+                        tmp.Clear();
                     }
                 }
+                tmp.Add(package);
             }
         }
 

src/JT1078.SignalR.Test/Startup.cs

@@ -50,6 +50,8 @@ namespace JT1078.SignalR.Test
             {
                 app.UseDeveloperExceptionPage();
             }
+            app.UseDefaultFiles();
+            app.UseStaticFiles();
             app.UseRouting();
             app.UseCors("CorsPolicy");
             app.UseAuthorization();