将1078转fmp4_9

doc/ffmpeginfo.txt

@@ -8,4 +8,6 @@ ffmpeg -i demo.264 -vcodec copy -f mp4 -movflags frag_keyframe+empty_moov fragme
 
 ffmpeg -i ipc.264 -vcodec copy -f mp4 -movflags frag_keyframe+empty_moov ipc_fragmented_demo.mp4
 
+ffmpeg -i jt1078_3.h264 -c:v copy -f mp4 -movflags empty_moov+default_base_moof+frag_keyframe fragmented_base_moof_demo.mp4
+
 chrome://media-internals/

src/JT1078.FMp4.Test/H264/index.html

@@ -96,28 +96,28 @@
             // otherwise insert it to queue
             var memview = new Uint8Array(arr);
             if (verbose) { console.log("got", arr.byteLength, "bytes.  Values=", memview[0], memview[1], memview[2], memview[3], memview[4]); }
-            res = getBox(memview, 0);
-            main_length = res[0]; name = res[1]; // this boxes length and name
-            if ((name == "ftyp") && (pass == 0)) {
-                pass = pass + 1;
-                console.log("got ftyp");
-            }
-            else if ((name == "moov") && (pass == 1)) {
-                pass = pass + 1;
-                console.log("got moov");
-            }
-            else if ((name == "moof") && (pass == 2)) {
-                if (hasFirstSampleFlag(memview)) {
-                    pass = pass + 1;
-                    console.log("got that special moof");
-                }
-                else {
-                    return;
-                }
-            }
-            else if (pass < 3) {
-                return;
-            }
+            //res = getBox(memview, 0);
+            //main_length = res[0]; name = res[1]; // this boxes length and name
+            //if ((name == "ftyp") && (pass == 0)) {
+            //    pass = pass + 1;
+            //    console.log("got ftyp");
+            //}
+            //else if ((name == "moov") && (pass == 1)) {
+            //    pass = pass + 1;
+            //    console.log("got moov");
+            //}
+            //else if ((name == "moof") && (pass == 2)) {
+            //    if (hasFirstSampleFlag(memview)) {
+            //        pass = pass + 1;
+            //        console.log("got that special moof");
+            //    }
+            //    else {
+            //        return;
+            //    }
+            //}
+            //else if (pass < 3) {
+            //    return;
+            //}
             // keep the latency to minimum
             let latest = stream_live.duration;
             if ((stream_live.duration >= buffering_sec) &&
@@ -132,8 +132,9 @@
             data = arr;
             if (!stream_started) {
                 if (verbose) { console.log("Streaming started: ", memview[0], memview[1], memview[2], memview[3], memview[4]); }
-                source_buffer.appendBuffer(data);
                 stream_started = true;
+                source_buffer.appendBuffer(data);
+                
                 cc = cc + 1;
                 return;
             }
@@ -173,7 +174,7 @@
                            .build();
             ws.on("video", (message) => {
                 var buff=base64ToArrayBuffer(message);
-                console.log(buff);
+                //console.log(buff);
                 putPacket(buff);
             });
             ws.start().catch(err => console.error(err));

src/JT1078.FMp4.Test/JT1078ToFMp4Box_Test.cs

@@ -451,13 +451,13 @@ namespace JT1078.FMp4.Test
             var nalus1 = h264Decoder.ParseNALU(package1);
             var moov = fMp4Encoder.EncoderMoovBox(nalus1, package1.Bodies.Length);
             fileStream.Write(moov);
-            int moofOffset = ftyp.Length + moov.Length;
-            var flag = package1.Label3.DataType == Protocol.Enums.JT1078DataType.视频I帧 ? 1u : 0u;
-            var otherMoofBuffer = fMp4Encoder.EncoderMoofBox(nalus1, package1.Bodies.Length, package1.Timestamp, flag);
             foreach (var package in packages)
             {
                 var otherNalus = h264Decoder.ParseNALU(package);
+                var flag = package.Label3.DataType == Protocol.Enums.JT1078DataType.视频I帧 ? 1u : 0u;
+                var otherMoofBuffer = fMp4Encoder.EncoderMoofBox(otherNalus, package.Bodies.Length, package.Timestamp, package.LastIFrameInterval, flag);
                 var otherMdatBuffer = fMp4Encoder.EncoderMdatBox(otherNalus, package.Bodies.Length);
+                fileStream.Write(otherMoofBuffer);
                 fileStream.Write(otherMdatBuffer);
             }
             fileStream.Close();

src/JT1078.FMp4/Boxs/SampleDependencyTypeBox.cs

@@ -30,15 +30,13 @@ namespace JT1078.FMp4
             WriterFullBoxToBuffer(ref writer);
             if(SampleDependencyTypes!=null && SampleDependencyTypes.Count > 0)
             {
-                foreach(var item in SampleDependencyTypes)
+                foreach (var item in SampleDependencyTypes)
                 {
-                    writer.WriteByte(item.IsLeading);
-                    writer.WriteByte(item.SampleDependsOn);
-                    writer.WriteByte(item.SampleIsDependedOn);
-                    writer.WriteByte(item.SampleHasRedundancy);
-                    writer.WriteByte(item.DegradPrio);
-                    writer.WriteByte(item.IsNonSync);
-                    writer.WriteByte(item.PaddingValue);
+                    writer.WriteByte((byte)(item.IsLeading<<2 | 
+                        item.SampleDependsOn | 
+                        item.SampleIsDependedOn <<6| 
+                        item.SampleHasRedundancy << 4 |
+                        item.IsNonSync));
                 }
             }
             End(ref writer);
@@ -50,9 +48,9 @@ namespace JT1078.FMp4
             public byte SampleDependsOn { get; set; }
             public byte SampleIsDependedOn { get; set; }
             public byte SampleHasRedundancy { get; set; }
-            public byte DegradPrio { get; set; }
+            //public byte DegradPrio { get; set; }
             public byte IsNonSync { get; set; }
-            public byte PaddingValue { get; set; }
+            //public byte PaddingValue { get; set; }
         }
     }
 }

src/JT1078.FMp4/Boxs/SegmentTypeBox.cs

@@ -0,0 +1,51 @@
+using JT1078.FMp4.Interfaces;
+using JT1078.FMp4.MessagePack;
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace JT1078.FMp4
+{
+    /// <summary>
+    /// styp
+    /// </summary>
+    public class SegmentTypeBox : Mp4Box, IFMp4MessagePackFormatter
+    {
+        /// <summary>
+        /// styp
+        /// </summary>
+        public SegmentTypeBox() : base("styp")
+        {
+        }
+        /// <summary>
+        ///因为兼容性一般可以分为推荐兼容性和默认兼容性。这里 major_brand 就相当于是推荐兼容性。通常，在 Web 中解码，一般而言都是使用 isom 这个万金油即可。如果是需要特定的格式，可以自行定义。
+        /// 4位
+        /// </summary>
+        public string MajorBrand { get; set; }
+        /// <summary>
+        /// 最低兼容版本
+        /// 4位
+        /// </summary>
+        public string MinorVersion { get; set; } = "\0\0\0\0";
+        /// <summary>
+        /// 和MajorBrand类似，通常是针对 MP4 中包含的额外格式，比如，AVC，AAC 等相当于的音视频解码格式。
+        /// 4位*n
+        /// </summary>
+        public List<string> CompatibleBrands { get; set; } = new List<string>();
+
+        public void ToBuffer(ref FMp4MessagePackWriter writer)
+        {
+            Start(ref writer);
+            writer.WriteASCII(MajorBrand);
+            writer.WriteASCII(MinorVersion);
+            if (CompatibleBrands != null && CompatibleBrands.Count > 0)
+            {
+                foreach (var item in CompatibleBrands)
+                {
+                    writer.WriteASCII(item);
+                }
+            }
+            End(ref writer);
+        }
+    }
+}

src/JT1078.FMp4/FMp4Encoder.cs

@@ -37,6 +37,31 @@ namespace JT1078.FMp4
             h264Decoder = new H264Decoder();
         }
 
+        public byte[] EncoderStypBox()
+        {
+            byte[] buffer = FMp4ArrayPool.Rent(4096);
+            FMp4MessagePackWriter writer = new FMp4MessagePackWriter(buffer);
+            try
+            {
+                //styp
+                SegmentTypeBox stypTypeBox = new SegmentTypeBox();
+                stypTypeBox.MajorBrand = "msdh";
+                stypTypeBox.MinorVersion = "\0\0\0\0";
+                stypTypeBox.CompatibleBrands.Add("isom");
+                stypTypeBox.CompatibleBrands.Add("mp42");
+                stypTypeBox.CompatibleBrands.Add("msdh");
+                stypTypeBox.CompatibleBrands.Add("nsix");
+                stypTypeBox.CompatibleBrands.Add("iso5");
+                stypTypeBox.CompatibleBrands.Add("iso6");
+                stypTypeBox.ToBuffer(ref writer);
+                var data = writer.FlushAndGetArray();
+                return data;
+            }
+            finally
+            {
+                FMp4ArrayPool.Return(buffer);
+            }
+        }
 
         /// <summary>
         /// 编码ftyp盒子
@@ -50,14 +75,22 @@ namespace JT1078.FMp4
             {
                 //ftyp
                 FileTypeBox fileTypeBox = new FileTypeBox();
-                fileTypeBox.MajorBrand = "msdh";
-                fileTypeBox.MinorVersion = "\0\0\0\0";
+                fileTypeBox.MajorBrand = "isom";
+                fileTypeBox.MinorVersion = "\0\0\u0002\0";
                 fileTypeBox.CompatibleBrands.Add("isom");
-                fileTypeBox.CompatibleBrands.Add("mp42");
-                fileTypeBox.CompatibleBrands.Add("msdh");
-                fileTypeBox.CompatibleBrands.Add("nsix");
+                fileTypeBox.CompatibleBrands.Add("iso2");
+                fileTypeBox.CompatibleBrands.Add("avc1");
+                fileTypeBox.CompatibleBrands.Add("mp41");
                 fileTypeBox.CompatibleBrands.Add("iso5");
-                fileTypeBox.CompatibleBrands.Add("iso6");
+
+                //FileTypeBox fileTypeBox = new FileTypeBox();
+                //fileTypeBox.MajorBrand = "iso5";
+                //fileTypeBox.MinorVersion = "\0\0\u0002\0";
+                //fileTypeBox.CompatibleBrands.Add("iso5");
+                //fileTypeBox.CompatibleBrands.Add("iso6");
+                //fileTypeBox.CompatibleBrands.Add("mp41");
+                //fileTypeBox.ToBuffer(ref writer);
+
                 fileTypeBox.ToBuffer(ref writer);
                 var data = writer.FlushAndGetArray();
                 return data;
@@ -132,35 +165,35 @@ namespace JT1078.FMp4
                 avc1.AVCConfigurationBox.SPSs = new List<byte[]>() { spsNALU.RawData };
                 movieBox.TrackBox.MediaBox.MediaInformationBox.SampleTableBox.SampleDescriptionBox.SampleEntries.Add(avc1);
                 movieBox.TrackBox.MediaBox.MediaInformationBox.SampleTableBox.TimeToSampleBox = new TimeToSampleBox() { 
-                     TimeToSampleInfos=new List<TimeToSampleBox.TimeToSampleInfo>
-                     {
-                         new TimeToSampleBox.TimeToSampleInfo
-                         {
-                              SampleCount=0,
-                              SampleDelta=0
-                         }
-                     }
+                     //TimeToSampleInfos=new List<TimeToSampleBox.TimeToSampleInfo>
+                     //{
+                     //    new TimeToSampleBox.TimeToSampleInfo
+                     //    {
+                     //         SampleCount=0,
+                     //         SampleDelta=0
+                     //    }
+                     //}
                 };
                 movieBox.TrackBox.MediaBox.MediaInformationBox.SampleTableBox.SampleToChunkBox = new SampleToChunkBox() { 
-                     SampleToChunkInfos=new List<SampleToChunkBox.SampleToChunkInfo>()
-                     {
-                          new SampleToChunkBox.SampleToChunkInfo
-                          {
+                     //SampleToChunkInfos=new List<SampleToChunkBox.SampleToChunkInfo>()
+                     //{
+                     //     new SampleToChunkBox.SampleToChunkInfo
+                     //     {
                                 
-                          }
-                     }
+                     //     }
+                     //}
                 };
                 movieBox.TrackBox.MediaBox.MediaInformationBox.SampleTableBox.SampleSizeBox = new SampleSizeBox() {
-                    EntrySize = new List<uint>()
-                    { 
-                        0
-                    }
+                    //EntrySize = new List<uint>()
+                    //{ 
+                    //    0
+                    //}
                 };
                 movieBox.TrackBox.MediaBox.MediaInformationBox.SampleTableBox.ChunkOffsetBox = new ChunkOffsetBox() { 
-                     ChunkOffset=new List<uint>()
-                     {
-                         0
-                     }
+                     //ChunkOffset=new List<uint>()
+                     //{
+                     //    0
+                     //}
                 };
                 movieBox.MovieExtendsBox = new MovieExtendsBox();
                 movieBox.MovieExtendsBox.TrackExtendsBoxs = new List<TrackExtendsBox>();
@@ -186,7 +219,7 @@ namespace JT1078.FMp4
         /// 编码Moof盒子
         /// </summary>
         /// <returns></returns>
-        public byte[] EncoderMoofBox(List<H264NALU> nalus, int naluLength,ulong timestamp, uint keyframeFlag,uint moofOffset=0)
+        public byte[] EncoderMoofBox(List<H264NALU> nalus, int naluLength,ulong timestamp,uint frameInterval, uint keyframeFlag,int moofOffset=0)
         {
             byte[] buffer = FMp4ArrayPool.Rent(naluLength + 4096);
             FMp4MessagePackWriter writer = new FMp4MessagePackWriter(buffer);
@@ -198,37 +231,60 @@ namespace JT1078.FMp4
                 movieFragmentBox.TrackFragmentBox = new TrackFragmentBox();
                 //0x39 写文件
                 //0x02 分段
-                movieFragmentBox.TrackFragmentBox.TrackFragmentHeaderBox = new TrackFragmentHeaderBox(2);
+                movieFragmentBox.TrackFragmentBox.TrackFragmentHeaderBox = new TrackFragmentHeaderBox(0x20038);
                 movieFragmentBox.TrackFragmentBox.TrackFragmentHeaderBox.TrackID = 1;
                 movieFragmentBox.TrackFragmentBox.TrackFragmentHeaderBox.DefaultSampleDuration = 48000;
                 movieFragmentBox.TrackFragmentBox.TrackFragmentHeaderBox.DefaultSampleSize = (uint)naluLength;
                 movieFragmentBox.TrackFragmentBox.TrackFragmentHeaderBox.DefaultSampleFlags = 0x1010000;
                 movieFragmentBox.TrackFragmentBox.TrackFragmentBaseMediaDecodeTimeBox = new TrackFragmentBaseMediaDecodeTimeBox();
+                //movieFragmentBox.TrackFragmentBox.SampleDependencyTypeBox = new SampleDependencyTypeBox()
+                //{
+                //    SampleDependencyTypes = new List<SampleDependencyTypeBox.SampleDependencyType>()
+                //};
                 //trun
                 //0x39 写文件
                 //0x02 分段
-                uint flag = 0u;
+                //uint flag = 0x000200 | 0x000800 | 0x000400 | 0x000100;
+                uint flag = 4u;
+
+                //var sdtp = new SampleDependencyTypeBox.SampleDependencyType();
+                //if (keyframeFlag==1)
+                //{
+                //    sdtp.SampleDependsOn = 2;
+                //    sdtp.SampleIsDependedOn = 1;
+                //}
+                //else
+                //{
+                //    sdtp.SampleDependsOn = 1;
+                //    sdtp.SampleIsDependedOn = 0;
+                //}
+
                 if (!first)
                 {
-                    flag = 4u;
+                    //sdtp.IsLeading = 1;
+                    //flag = 4u;
                     movieFragmentBox.TrackFragmentBox.TrackFragmentBaseMediaDecodeTimeBox.BaseMediaDecodeTime = 0;
-                    movieFragmentBox.TrackFragmentBox.TrackRunBox = new TrackRunBox(flags: flag);
+                    movieFragmentBox.TrackFragmentBox.TrackRunBox = new TrackRunBox(flags: 0x205);
                     first = true;
                 }
                 else
                 {
-                    flag = 0x000400;
-                    movieFragmentBox.TrackFragmentBox.TrackFragmentBaseMediaDecodeTimeBox.BaseMediaDecodeTime = timestamp * 1000;
-                    movieFragmentBox.TrackFragmentBox.TrackRunBox = new TrackRunBox(flags: flag);
+                    //flag = 0x000400;
+                    movieFragmentBox.TrackFragmentBox.TrackFragmentBaseMediaDecodeTimeBox.BaseMediaDecodeTime = BaseMediaDecodeTime;
+                    movieFragmentBox.TrackFragmentBox.TrackRunBox = new TrackRunBox(flags: 0x205);
+                    BaseMediaDecodeTime += BaseMediaDecodeTime;
                 }
-                movieFragmentBox.TrackFragmentBox.TrackRunBox.FirstSampleFlags = 0;
+                //movieFragmentBox.TrackFragmentBox.SampleDependencyTypeBox.SampleDependencyTypes.Add(sdtp);
+
+                movieFragmentBox.TrackFragmentBox.TrackRunBox.FirstSampleFlags = 33554432;
                 movieFragmentBox.TrackFragmentBox.TrackRunBox.TrackRunInfos = new List<TrackRunBox.TrackRunInfo>();
-                movieFragmentBox.TrackFragmentBox.TrackRunBox.TrackRunInfos.Add(new TrackRunBox.TrackRunInfo());
-         
+
+                //movieFragmentBox.TrackFragmentBox.TrackRunBox.TrackRunInfos.Add(new TrackRunBox.TrackRunInfo());
                 movieFragmentBox.TrackFragmentBox.TrackRunBox.TrackRunInfos.Add(new TrackRunBox.TrackRunInfo()
                 {
+                    SampleDuration= frameInterval,
                     SampleSize = (uint)naluLength,
-                    //SampleCompositionTimeOffset = package.Label3.DataType == JT1078DataType.视频I帧 ? package.LastIFrameInterval : package.LastFrameInterval,
+                    SampleCompositionTimeOffset = frameInterval,
                     SampleFlags = flag
                 });
                 
@@ -366,6 +422,8 @@ namespace JT1078.FMp4
 
         bool first = false;
 
+        ulong BaseMediaDecodeTime = 2160000;
+
         /// <summary>
         /// 编码其他视频数据盒子
         /// </summary>

src/JT1078.FMp4/JT1078.FMp4.xml

@@ -712,6 +712,34 @@
             4byte 32 - 28
             </summary>
         </member>
+        <member name="T:JT1078.FMp4.SegmentTypeBox">
+            <summary>
+            styp
+            </summary>
+        </member>
+        <member name="M:JT1078.FMp4.SegmentTypeBox.#ctor">
+            <summary>
+            styp
+            </summary>
+        </member>
+        <member name="P:JT1078.FMp4.SegmentTypeBox.MajorBrand">
+             <summary>
+            因为兼容性一般可以分为推荐兼容性和默认兼容性。这里 major_brand 就相当于是推荐兼容性。通常，在 Web 中解码，一般而言都是使用 isom 这个万金油即可。如果是需要特定的格式，可以自行定义。
+             4位
+             </summary>
+        </member>
+        <member name="P:JT1078.FMp4.SegmentTypeBox.MinorVersion">
+            <summary>
+            最低兼容版本
+            4位
+            </summary>
+        </member>
+        <member name="P:JT1078.FMp4.SegmentTypeBox.CompatibleBrands">
+            <summary>
+            和MajorBrand类似，通常是针对 MP4 中包含的额外格式，比如，AVC，AAC 等相当于的音视频解码格式。
+            4位*n
+            </summary>
+        </member>
         <member name="P:JT1078.FMp4.StereoVideoBox.Reserved">
              <summary>
             4btye 32 - 30
@@ -1239,7 +1267,7 @@
             </summary>
             <returns></returns>
         </member>
-        <member name="M:JT1078.FMp4.FMp4Encoder.EncoderMoofBox(System.Collections.Generic.List{JT1078.Protocol.H264.H264NALU},System.Int32,System.UInt64,System.UInt32,System.UInt32)">
+        <member name="M:JT1078.FMp4.FMp4Encoder.EncoderMoofBox(System.Collections.Generic.List{JT1078.Protocol.H264.H264NALU},System.Int32,System.UInt64,System.UInt32,System.UInt32,System.Int32)">
             <summary>
             编码Moof盒子
             </summary>

src/JT1078.SignalR.Test/Services/ToWebSocketService.cs

@@ -46,7 +46,7 @@ namespace JT1078.SignalR.Test.Services
             this.wsSession = wsSession;
         }
 
-        public Queue<byte[]> q = new Queue<byte[]>();
+        public List<byte[]> q = new List<byte[]>();
 
         public void a()
         {
@@ -65,23 +65,40 @@ namespace JT1078.SignalR.Test.Services
                     packages.Add(packageMerge);
                 }
             }
+            List<byte[]> first = new List<byte[]>();
+            //var styp = fMp4Encoder.EncoderStypBox();
+            //first.Add(styp);
+            //q.Enqueue(styp);
             var ftyp = fMp4Encoder.EncoderFtypBox();
-            q.Enqueue(ftyp);
+            //q.Enqueue(ftyp);
+            first.Add(ftyp);
             var package1 = packages[0];
             var nalus1 = h264Decoder.ParseNALU(package1);
             var moov = fMp4Encoder.EncoderMoovBox(nalus1, package1.Bodies.Length);
-            q.Enqueue(moov);
-            var flag = package1.Label3.DataType == Protocol.Enums.JT1078DataType.视频I帧 ? 1u : 0u;
-            var moofBuffer = fMp4Encoder.EncoderMoofBox(nalus1, package1.Bodies.Length, package1.Timestamp, flag);
-            q.Enqueue(moofBuffer);
+            //q.Enqueue(moov);
+            first.Add(moov);
+            q.Add(first.SelectMany(s=>s).ToArray());
+            List<int> filter = new List<int>() { 6,7,8};
             foreach (var package in packages)
             {
+                List<byte[]> other = new List<byte[]>();
                 var otherNalus = h264Decoder.ParseNALU(package);
-                var otherMdatBuffer = fMp4Encoder.EncoderMdatBox(otherNalus, package.Bodies.Length);
-                q.Enqueue(otherMdatBuffer);
+                var filterNalus = otherNalus.Where(w => !filter.Contains(w.NALUHeader.NalUnitType)).ToList();
+                var flag = package.Label3.DataType == Protocol.Enums.JT1078DataType.视频I帧 ? 1u : 0u;
+                var len = filterNalus.Sum(s => s.RawData.Length);
+                var len1 = otherNalus.Sum(s => s.RawData.Length);
+                var moofBuffer = fMp4Encoder.EncoderMoofBox(filterNalus, len, package.Timestamp, package.LastIFrameInterval, flag);
+                //q.Enqueue(moofBuffer);
+                other.Add(moofBuffer);
+                var otherMdatBuffer = fMp4Encoder.EncoderMdatBox(filterNalus, len);
+                //q.Enqueue(otherMdatBuffer);
+                other.Add(otherMdatBuffer);
+                q.Add(other.SelectMany(s => s).ToArray());
             }
         }
 
+        public Dictionary<string,int> flag = new Dictionary<string, int>();
+
         protected async override Task ExecuteAsync(CancellationToken stoppingToken)
         {
             a();
@@ -89,11 +106,22 @@ namespace JT1078.SignalR.Test.Services
             {
                 try
                 {
-                    if (wsSession.GetCount() > 0)
+                    foreach(var session in wsSession.GetAll())
                     {
-                        if (q.Count > 0)
+                        if (flag.ContainsKey(session))
+                        {
+                            var len = flag[session];
+                            if (q.Count < len)
+                            {
+                                break;
+                            }
+                            await _hubContext.Clients.Client(session).SendAsync("video", q[len], stoppingToken);
+                            flag[session] = ++len;
+                        }
+                        else
                         {
-                            await _hubContext.Clients.All.SendAsync("video", q.Dequeue(), stoppingToken);
+                            await _hubContext.Clients.Client(session).SendAsync("video", q[0], stoppingToken);
+                            flag.Add(session, 1);
                         }
                     }
                 }

src/JT1078.SignalR.Test/Services/WsSession.cs

@@ -29,5 +29,10 @@ namespace JT1078.SignalR.Test.Services
         {
             sessions.TryRemove(connectionId,out _);
         }
+
+        public List<string> GetAll()
+        {
+            return sessions.Keys.ToList();
+        }
     }
 }