From 29c9a9f988e90b95d0ed98fe3a9f88d37181dd8b Mon Sep 17 00:00:00 2001
From: "SmallChi(Koike)" <564952747@qq.com>
Date: Sun, 5 Jun 2022 23:51:30 +0800
Subject: [PATCH] =?UTF-8?q?=E8=B0=83=E6=95=B4ws-fmp4=E6=92=AD=E6=94=BE?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 README.md                                     |   2 +-
 .../Configs/nlog.Win32NT.config               |   4 +-
 .../Program.cs                                |   2 +-
 .../JT1078FMp4NormalMsgHostedService.cs       | 107 ++-----
 .../appsettings.json                          |   5 +-
 .../wwwroot/fmp4_demo/index.html              | 284 +++++++-----------
 src/JT1078.Gateway.sln                        |   7 +
 src/JT1078.Gateway/JT1078.Gateway.csproj      |   7 +-
 8 files changed, 151 insertions(+), 267 deletions(-)

diff --git a/README.md b/README.md
index c3a6453..3066bb9 100644
--- a/README.md
+++ b/README.md
@@ -15,7 +15,7 @@
 | --- | ---| --- |---|---|
 | flv  | 😀| ☹ |😀|http-flv、ws-flv|
 | m3u8 | 😀| ☹ |😀|http|
-| fmp4 | 😀| ☹ |☹|http-fmp4、ws-fmp4|
+| fmp4 | 😀| ☹ |😀(部分设备可用)|http-fmp4[X]、ws-fmp4[✔]|
 
 ## NuGet安装
 
diff --git a/src/JT1078.Gateway.Tests/JT1078.Gateway.TestNormalHosting/Configs/nlog.Win32NT.config b/src/JT1078.Gateway.Tests/JT1078.Gateway.TestNormalHosting/Configs/nlog.Win32NT.config
index 78f7b08..67c4513 100644
--- a/src/JT1078.Gateway.Tests/JT1078.Gateway.TestNormalHosting/Configs/nlog.Win32NT.config
+++ b/src/JT1078.Gateway.Tests/JT1078.Gateway.TestNormalHosting/Configs/nlog.Win32NT.config
@@ -41,8 +41,8 @@
     </targets>
     <rules>
       <logger name="*" minlevel="Error" writeTo="all,console"/>
-      <logger name="JT1078.Gateway.JT1078TcpServer" minlevel="Trace" writeTo="JT1078TcpServer,console"/>
-      <logger name="JT1078.Gateway.JT1078UdpServer" minlevel="Trace" writeTo="JT1078UdpServer,console"/>
+      <logger name="JT1078.Gateway.JT1078TcpServer" minlevel="Info" writeTo="JT1078TcpServer,console"/>
+      <logger name="JT1078.Gateway.JT1078UdpServer" minlevel="Info" writeTo="JT1078UdpServer,console"/>
       <logger name="JT1078.Gateway.JT1078Logging" minlevel="Trace" writeTo="JT1078Logging,console"/>
     </rules>
   </nlog>
\ No newline at end of file
diff --git a/src/JT1078.Gateway.Tests/JT1078.Gateway.TestNormalHosting/Program.cs b/src/JT1078.Gateway.Tests/JT1078.Gateway.TestNormalHosting/Program.cs
index 61fd6fb..bb33ae3 100644
--- a/src/JT1078.Gateway.Tests/JT1078.Gateway.TestNormalHosting/Program.cs
+++ b/src/JT1078.Gateway.Tests/JT1078.Gateway.TestNormalHosting/Program.cs
@@ -29,7 +29,7 @@ namespace JT1078.Gateway.TestNormalHosting
                 })
                 .ConfigureLogging((context, logging) =>
                 {
-                    logging.SetMinimumLevel(LogLevel.Trace);
+                    //logging.SetMinimumLevel(LogLevel.Trace);
                     Console.WriteLine($"Environment.OSVersion.Platform:{Environment.OSVersion.Platform.ToString()}");
                     NLog.LogManager.LoadConfiguration($"Configs/nlog.{Environment.OSVersion.Platform.ToString()}.config");
                     logging.AddNLog(new NLogProviderOptions { CaptureMessageTemplates = true, CaptureMessageProperties = true });
diff --git a/src/JT1078.Gateway.Tests/JT1078.Gateway.TestNormalHosting/Services/JT1078FMp4NormalMsgHostedService.cs b/src/JT1078.Gateway.Tests/JT1078.Gateway.TestNormalHosting/Services/JT1078FMp4NormalMsgHostedService.cs
index 7a1f245..888a2a6 100644
--- a/src/JT1078.Gateway.Tests/JT1078.Gateway.TestNormalHosting/Services/JT1078FMp4NormalMsgHostedService.cs
+++ b/src/JT1078.Gateway.Tests/JT1078.Gateway.TestNormalHosting/Services/JT1078FMp4NormalMsgHostedService.cs
@@ -26,8 +26,7 @@ namespace JT1078.Gateway.TestNormalHosting.Services
         private MessageDispatchDataService messageDispatchDataService;
         private ConcurrentDictionary<string, FMp4AVContext> avFrameDict;
         private H264Decoder H264Decoder;
-        List<NalUnitType> NaluFilter;
-        BlockingCollection<(string SIM, byte ChannelNo,byte[]FirstBuffer, byte[] OtherBuffer)> FMp4Blocking;
+        BlockingCollection<(string SIM, byte ChannelNo,byte[]FirstBuffer, byte[] AVFrameInfoBuffer, byte[] OtherBuffer)> FMp4Blocking;
         public JT1078FMp4NormalMsgHostedService(
             MessageDispatchDataService messageDispatchDataService,
             ILoggerFactory loggerFactory,
@@ -41,12 +40,7 @@ namespace JT1078.Gateway.TestNormalHosting.Services
             H264Decoder= h264Decoder;
             this.messageDispatchDataService = messageDispatchDataService;
             avFrameDict = new ConcurrentDictionary<string, FMp4AVContext>();
-            FMp4Blocking=new BlockingCollection<(string SIM, byte ChannelNo, byte[] FirstBuffer, byte[] OtherBuffer)>();
-            NaluFilter = new List<NalUnitType>();
-            NaluFilter.Add(NalUnitType.SEI);
-            NaluFilter.Add(NalUnitType.PPS);
-            NaluFilter.Add(NalUnitType.SPS);
-            NaluFilter.Add(NalUnitType.AUD);
+            FMp4Blocking=new BlockingCollection<(string SIM, byte ChannelNo, byte[] FirstBuffer, byte[] AVFrameInfoBuffer, byte[] OtherBuffer)>();
         }
 
         public Task StartAsync(CancellationToken cancellationToken)
@@ -57,74 +51,32 @@ namespace JT1078.Gateway.TestNormalHosting.Services
                     var data = await messageDispatchDataService.FMp4Channel.Reader.ReadAsync();
                     try
                     {
-                        if (Logger.IsEnabled(LogLevel.Debug))
+                        //if (Logger.IsEnabled(LogLevel.Debug))
+                        //{
+                        //    Logger.LogDebug(JsonSerializer.Serialize(HttpSessionManager.GetAll()));
+                        //    Logger.LogDebug($"{data.SIM},{data.SN},{data.LogicChannelNumber},{data.Label3.DataType.ToString()},{data.Label3.SubpackageType.ToString()},{data.Bodies.ToHexString()}");
+                        //}
+                        bool keyframe = data.Label3.DataType == Protocol.Enums.JT1078DataType.视频I帧;
+                        JT1078AVFrame avframe = H264Decoder.ParseAVFrame(data);
+                        if (avframe.Nalus!= null && avframe.Nalus.Count>0)
                         {
-                            Logger.LogDebug(JsonSerializer.Serialize(HttpSessionManager.GetAll()));
-                            Logger.LogDebug($"{data.SIM},{data.SN},{data.LogicChannelNumber},{data.Label3.DataType.ToString()},{data.Label3.SubpackageType.ToString()},{data.Bodies.ToHexString()}");
-                        }
-                        List<H264NALU> h264NALUs = H264Decoder.ParseNALU(data);
-                        if (h264NALUs!=null && h264NALUs.Count>0)
-                        {
-                            if(!avFrameDict.TryGetValue(data.GetKey(),out FMp4AVContext cacheFrame))
+                            if(!avFrameDict.TryGetValue(data.GetAVKey(),out FMp4AVContext cacheFrame))
                             {
                                 cacheFrame=new FMp4AVContext();
-                                avFrameDict.TryAdd(data.GetKey(), cacheFrame);
+                                avFrameDict.TryAdd(data.GetAVKey(), cacheFrame);
                             }
-                            foreach(var nalu in h264NALUs)
+                            if(keyframe)
                             {
-                                if (NaluFilter.Contains(nalu.NALUHeader.NalUnitType))
+                                if(avframe.SPS!=null && avframe.PPS != null)
                                 {
-                                    if (nalu.NALUHeader.NalUnitType== NalUnitType.SPS)
-                                    {
-                                        cacheFrame.SPSNalu=nalu;
-                                    }
-                                    else if (nalu.NALUHeader.NalUnitType== NalUnitType.PPS)
-                                    {
-                                        cacheFrame.PPSNalu=nalu;
-                                    }
-                                }
-                                else
-                                {
-                                    cacheFrame.NALUs.Add(nalu);
+                                    cacheFrame.AVFrameInfoBuffer = JsonSerializer.SerializeToUtf8Bytes(
+                                        new { Codecs = avframe .ToCodecs(), Width = avframe .Width, Height =avframe.Height});
+                                    cacheFrame.FirstCacheBuffer = FM4Encoder.FirstVideoBox(avframe);
                                 }
                             }
-                            if (cacheFrame.NALUs.Count>1)
+                            if (cacheFrame.FirstCacheBuffer != null)
                             {
-                                if (cacheFrame.FirstCacheBuffer==null)
-                                {
-                                    cacheFrame.FirstCacheBuffer=FM4Encoder.FirstVideoBox(cacheFrame.SPSNalu, cacheFrame.PPSNalu);
-                                }
-                                List<H264NALU> tmp = new List<H264NALU>();
-                                int i = 0;
-                                foreach (var item in cacheFrame.NALUs)
-                                {
-                                    if (item.NALUHeader.KeyFrame)
-                                    {
-                                        if (tmp.Count>0)
-                                        {
-                                            FMp4Blocking.Add((data.SIM, data.LogicChannelNumber, cacheFrame.FirstCacheBuffer, FM4Encoder.OtherVideoBox(tmp)));
-                                            i+=tmp.Count;
-                                            tmp.Clear();
-                                        }
-                                        tmp.Add(item);
-                                        i+=tmp.Count;
-                                        FMp4Blocking.Add((data.SIM, data.LogicChannelNumber, cacheFrame.FirstCacheBuffer, FM4Encoder.OtherVideoBox(tmp)));
-                                        tmp.Clear();
-                                        cacheFrame.PrevPrimaryNalu = item;
-                                        continue;
-                                    }
-                                    if (cacheFrame.PrevPrimaryNalu!=null) //第一帧I帧
-                                    {
-                                        if (tmp.Count>1)
-                                        {
-                                            FMp4Blocking.Add((data.SIM, data.LogicChannelNumber, cacheFrame.FirstCacheBuffer, FM4Encoder.OtherVideoBox(tmp)));
-                                            i+=tmp.Count;
-                                            tmp.Clear();
-                                        }
-                                        tmp.Add(item);
-                                    }
-                                }
-                                cacheFrame.NALUs.RemoveRange(0, i);
+                                FMp4Blocking.Add((data.SIM, data.LogicChannelNumber, cacheFrame.FirstCacheBuffer, cacheFrame.AVFrameInfoBuffer,FM4Encoder.OtherVideoBox(avframe.Nalus, data.GetAVKey(), keyframe)));
                             }
                         }
                     }
@@ -132,13 +84,18 @@ namespace JT1078.Gateway.TestNormalHosting.Services
                     {
                         Logger.LogError(ex, $"{data.SIM},{data.SN},{data.LogicChannelNumber},{data.Label3.DataType.ToString()},{data.Label3.SubpackageType.ToString()},{data.Bodies.ToHexString()}");
                     }
-
                 }
             });
             Task.Run(() => {
+                //var filepath = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "JT1078_7_4_4.mp4");
+                //if (File.Exists(filepath))
+                //{
+                //    File.Delete(filepath);
+                //}
+                //using var fileStream = new FileStream(filepath, FileMode.OpenOrCreate, FileAccess.Write);
                 try
                 {
-                    foreach(var item in FMp4Blocking.GetConsumingEnumerable(cancellationToken))
+                    foreach (var item in FMp4Blocking.GetConsumingEnumerable(cancellationToken))
                     {
                         var httpSessions = HttpSessionManager.GetAllBySimAndChannelNo(item.SIM.TrimStart('0'), item.ChannelNo);
                         var firstHttpSessions = httpSessions.Where(w => !w.FirstSend && (w.RTPVideoType == Metadata.RTPVideoType.Http_FMp4 || w.RTPVideoType == Metadata.RTPVideoType.Ws_FMp4)).ToList();
@@ -148,8 +105,11 @@ namespace JT1078.Gateway.TestNormalHosting.Services
                             //首帧
                             foreach (var session in firstHttpSessions)
                             {
-                                HttpSessionManager.SendAVData(session, item.FirstBuffer, true);
+                                HttpSessionManager.SendAVData(session, item.AVFrameInfoBuffer, true);
+                                HttpSessionManager.SendAVData(session, item.FirstBuffer, false);
+                                //fileStream.Write(item.FirstBuffer);
                                 HttpSessionManager.SendAVData(session, item.OtherBuffer, false);
+                                //fileStream.Write(item.OtherBuffer);
                             }
                         }
                         if (otherHttpSessions.Count > 0)
@@ -158,6 +118,7 @@ namespace JT1078.Gateway.TestNormalHosting.Services
                             foreach (var session in otherHttpSessions)
                             {
                                 HttpSessionManager.SendAVData(session, item.OtherBuffer, false);
+                                //fileStream.Write(item.OtherBuffer);
                             }
                         }
                     }
@@ -166,6 +127,7 @@ namespace JT1078.Gateway.TestNormalHosting.Services
                 {
 
                 }
+                //fileStream.Close();
             });
             return Task.CompletedTask;
         }
@@ -177,11 +139,8 @@ namespace JT1078.Gateway.TestNormalHosting.Services
 
         public class FMp4AVContext
         {
+            public byte[] AVFrameInfoBuffer { get; set; }
             public byte[] FirstCacheBuffer { get; set; }
-            public H264NALU PrevPrimaryNalu { get; set; }
-            public H264NALU SPSNalu { get; set; }
-            public H264NALU PPSNalu { get; set; }
-            public List<H264NALU> NALUs { get; set; } = new List<H264NALU>();
         }
     }
 }
diff --git a/src/JT1078.Gateway.Tests/JT1078.Gateway.TestNormalHosting/appsettings.json b/src/JT1078.Gateway.Tests/JT1078.Gateway.TestNormalHosting/appsettings.json
index 18c352a..f7ce160 100644
--- a/src/JT1078.Gateway.Tests/JT1078.Gateway.TestNormalHosting/appsettings.json
+++ b/src/JT1078.Gateway.Tests/JT1078.Gateway.TestNormalHosting/appsettings.json
@@ -1,14 +1,13 @@
 {
   "Logging": {
-    "IncludeScopes": false,
     "Debug": {
       "LogLevel": {
-        "Default": "Trace"
+        "Default": "Information"
       }
     },
     "Console": {
       "LogLevel": {
-        "Default": "Trace"
+        "Default": "Information"
       }
     }
   },
diff --git a/src/JT1078.Gateway.Tests/JT1078.Gateway.TestNormalHosting/wwwroot/fmp4_demo/index.html b/src/JT1078.Gateway.Tests/JT1078.Gateway.TestNormalHosting/wwwroot/fmp4_demo/index.html
index e0e6f1f..a5df29f 100644
--- a/src/JT1078.Gateway.Tests/JT1078.Gateway.TestNormalHosting/wwwroot/fmp4_demo/index.html
+++ b/src/JT1078.Gateway.Tests/JT1078.Gateway.TestNormalHosting/wwwroot/fmp4_demo/index.html
@@ -7,198 +7,116 @@
 </head>
 <body>
     <h1>MSE FMp4 Demo</h1>
-    <video id="stream_live" width="640" height="480" controls="false" autoplay="true"
-           muted="muted"
-           preload="auto">
+    <video id="stream_live" autoplay>
         浏览器不支持
     </video>
-    <ul id="messagesList"></ul>
     <script>
-        //var mimeCodec = 'video/mp4;codecs="avc1.4D0014, mp4a.40.2"';
-        // *** USER PARAMETERS ***
-        var verbose = true;
-        // var verbose = true; // enable for saturating the console ..
-        var buffering_sec = 1; // use some reasonable value
-        var buffering_sec_seek = buffering_sec * 0.9;
-        // ..seek the stream if it's this much away or
-        // from the last available timestamp
-        var buffering_sec_seek_distance = buffering_sec * 0.5;
-        // .. jump to this distance from the last avail. timestamp
-        // *** INTERNAL PARAMETERS ***
-        // set mimetype and codec
-        var mimeType = "video/mp4";
-        var codecs = "avc1.4D0014"; // https://wiki.whatwg.org/wiki/Video_type_parameters
-        // if your stream has audio, remember to include it in these definitions.. otherwise your mse goes sour
-        var codecPars = mimeType + ';codecs="' + codecs + '"';
-        var stream_started = false; // is the source_buffer updateend callback active nor not
-        // create media source instance
-        var ms = new MediaSource();
-        // queue for incoming media packets
-        var queue = [];
-        var stream_live; // the HTMLMediaElement (i.e. <video> element)
-        var ws; // websocket
-        var seeked = false; // have have seeked manually once ..
-        var cc = 0;
-        var source_buffer; // source_buffer instance
-        var pass = 0;
-        // *** MP4 Box manipulation functions ***
-        // taken from here: https://stackoverflow.com/questions/54186634/sending-periodic-metadata-in-fragmented-live-mp4-stream/
-        function toInt(arr, index) { // From bytes to big-endian 32-bit integer.  Input: Uint8Array, index
-            var dv = new DataView(arr.buffer, 0);
-            return dv.getInt32(index, false); // big endian
+        /* ref https://github.com/v354412101/wsPlayer.git */
+        function JT1078Player(videoId, wsUrl) {
+            this.videoId = videoId;
+            this.wsUrl = wsUrl;
+            this.ws = null;
+            this.verbose = true;
+            this.frameQueue = [];
         }
-        function toString(arr, fr, to) { // From bytes to string.  Input: Uint8Array, start index, stop index.
-            // https://developers.google.com/web/updates/2012/06/How-to-convert-ArrayBuffer-to-and-from-String
-            return String.fromCharCode.apply(null, arr.slice(fr, to));
-        }
-        function getBox(arr, i) { // input Uint8Array, start index
-            return [toInt(arr, i), toString(arr, i + 4, i + 8)]
-        }
-        function getSubBox(arr, box_name) { // input Uint8Array, box name
-            var i = 0;
-            res = getBox(arr, i);
-            main_length = res[0]; name = res[1]; // this boxes length and name
-            i = i + 8;
-            var sub_box = null;
-            while (i < main_length) {
-                res = getBox(arr, i);
-                l = res[0]; name = res[1];
-
-                if (box_name == name) {
-                    sub_box = arr.slice(i, i + l)
+        JT1078Player.prototype.open = function () {
+            let sourcebuffer = null;
+            this.ws = new WebSocket(this.wsUrl);
+            this.ws.binaryType = 'arraybuffer';
+            let firstMessage = true;
+            let initMediaSource = function (avframe) {
+                let video = document.getElementById(this.videoId);
+                let mediasource = new MediaSource();
+                video.src = URL.createObjectURL(mediasource);
+                let pre_pos = 0;
+                mediasource.onsourceopen = function () {
+                    sourcebuffer = mediasource.addSourceBuffer('video/mp4; codecs=' + avframe.Codecs);
+                    sourcebuffer.onupdateend = function () {
+                        let pos = video.currentTime;
+                        if (video.buffered.length > 0) {
+                            let start = video.buffered.start(video.buffered.length - 1);
+                            let end = video.buffered.end(video.buffered.length - 1);
+                            if (this.verbose)
+                                console.log("pos=" + pos + ",start=" + start + ",end=" + end);
+                            if (pos < start) {
+                                if (this.verbose)
+                                    console.log("set video.currentTime pos=" + pos + ",start=" + start + ",end=" + end);
+                                video.currentTime = start;
+                            }
+                            if (pos > end) {
+                                if (this.verbose)
+                                    console.warn("chase frame pos=" + pos + ",start=" + start + ",end=" + end);
+                                video.currentTime = start;
+                            }
+                            if (pos - pre_pos != 0 && end - pos > 3) {
+                                if (this.verbose)
+                                    console.log("set end video.currentTime pos=" + pos + ",start=" + start + ",end=" + end);
+                                video.currentTime = end;
+                            }
+                            for (let i = 0; i < video.buffered.length - 1; i++) {
+                                let prestart = video.buffered.start(i);
+                                let preend = video.buffered.end(i);
+                                if (!sourcebuffer.updating) {
+                                    sourcebuffer.remove(prestart, preend);
+                                }
+                            }
+                            if (pos - start > 10 && !sourcebuffer.updating) {
+                                if (this.verbose)
+                                    console.warn("remove start pos=" + pos + ",start=" + start + ",end=" + end);
+                                sourcebuffer.remove(0, pos - 3);
+                            }
+                            if (end - pos > 10 && !sourcebuffer.updating) {
+                                if (this.verbose)
+                                    console.warn("remove end pos=" + pos + ",start=" + start + ",end=" + end);
+                                sourcebuffer.remove(0, end - 3);
+                            }
+                        }
+                        pre_pos = pos;
+                    }
                 }
-                i = i + l;
-            }
-            return sub_box;
-        }
-        function hasFirstSampleFlag(arr) { // input Uint8Array
-            // [moof [mfhd] [traf [tfhd] [tfdt] [trun]]]
-            var traf = getSubBox(arr, "traf");
-            if (traf == null) { return false; }
-            var trun = getSubBox(traf, "trun");
-            if (trun == null) { return false; }
-            // ISO/IEC 14496-12:2012(E) .. pages 5 and 57
-            // bytes: (size 4), (name 4), (version 1 + tr_flags 3)
-            var flags = trun.slice(10, 13); // console.log(flags);
-            f = flags[1] & 4; // console.log(f);
-            return f == 4;
-        }
-        // consider these callbacks:
-        // - putPacket : called when websocket receives data
-        // - loadPacket : called when source_buffer is ready for more data
-        // Both operate on a common fifo
-        function putPacket(arr) {
-            // receives ArrayBuffer.  Called when websocket gets more data
-            // first packet ever to arrive: write directly to source_buffer
-            // source_buffer ready to accept: write directly to source_buffer
-            // otherwise insert it to queue
-            var memview = new Uint8Array(arr);
-            if (verbose) { console.log("got", arr.byteLength, "bytes.  Values=", memview[0], memview[1], memview[2], memview[3], memview[4]); }
-            res = getBox(memview, 0);
-            main_length = res[0]; name = res[1]; // this boxes length and name
-            // if ((name == "ftyp") && (pass == 0)) {
-            //    pass = pass + 1;
-            //    console.log("got ftyp");
-            // }
-            // else if ((name == "moov") && (pass == 1)) {
-            //    pass = pass + 1;
-            //    console.log("got moov");
-            // }
-            // else if ((name == "moof") && (pass == 2)) {
-            //    if (hasFirstSampleFlag(memview)) {
-            //        pass = pass + 1;
-            //        console.log("got that special moof");
-            //    }
-            //    else {
-            //        return;
-            //    }
-            // }
-            // else if (pass < 3) {
-            //    return;
-            // }
-            // keep the latency to minimum
-            // let latest = stream_live.duration;
-            // if ((stream_live.duration >= buffering_sec) && ((latest - stream_live.currentTime) > buffering_sec_seek)) {
-            //     console.log("seek from ", stream_live.currentTime, " to ", latest);
-            //     df = (stream_live.duration - stream_live.currentTime); // this much away from the last available frame
-            //     if ((df > buffering_sec_seek)) {
-            //         seek_to = stream_live.duration - buffering_sec_seek_distance;
-            //         stream_live.currentTime = seek_to;
-            //     }
-            // }
-            if (!source_buffer.updating) {
-                if (verbose) { console.log("Streaming started: ", memview[0], memview[1], memview[2], memview[3], memview[4]); }
-                stream_started = true;
-                source_buffer.appendBuffer(arr);          
-                cc = cc + 1;
-            }else{
-                queue.push(arr); // add to the end
-            }
-            if (verbose) { console.log("queue push:", queue.length); }
-        }
+            }.bind(this);
 
-        function loadPacket() { // called when source_buffer is ready for more
-            if (!source_buffer.updating) { // really, really ready
-                if (queue.length > 0) {
-                    var inp = queue.shift(); // pop from the beginning
-                    if (verbose) { console.log("queue pop:", queue.length); }
-                    var memview = new Uint8Array(inp);
-                    if (verbose) { console.log(" ==> writing buffer with", memview[0], memview[1], memview[2], memview[3]); }
-                    source_buffer.appendBuffer(inp);
-                    cc = cc + 1;
+            this.ws.onmessage = function (e) {
+                //首帧为音视频信息
+                if (firstMessage) {
+                    firstMessage = false;
+                    let avframe = JSON.parse(String.fromCharCode.apply(null, new Uint8Array(e.data)));
+                    if(this.verbose)
+                        console.log("avframe", avframe);
+                    initMediaSource(avframe);
+                    return;
                 }
-                // else { // the queue runs empty, so the next packet is fed directly
-                //     stream_started = false;
-                // }
-            }
-            // else { 
-            //     // so it was not?
-
-            // }
-        }
-
-        function opened() { // MediaSource object is ready to go
-            // https://developer.mozilla.org/en-US/docs/Web/API/MediaSource/duration
-            ms.duration = buffering_sec;
-            source_buffer = ms.addSourceBuffer(codecPars);
-            // https://developer.mozilla.org/en-US/docs/Web/API/source_buffer/mode
-            var myMode = source_buffer.mode;
-            source_buffer.mode = 'sequence';
-            // source_buffer.mode = 'segments';
-            source_buffer.addEventListener("updateend", loadPacket);
-            ws = new WebSocket("ws://49.235.89.102:15555/live.mp4?sim=40281815788&channel=1&token=123456");
-            
-            //ws = new WebSocket("ws://49.235.89.102:15555/live.mp4?sim=1901305037&channel=2&token=123456");
-            //ws = new WebSocket("ws://127.0.0.1:81/live/JT1078_8.live.mp4"); //创建WebSocket连接
-            ws.binaryType = 'arraybuffer';
-            ws.onmessage = function (e) {
-                //当客户端收到服务端发来的消息时，触发onmessage事件，参数e.data包含server传递过来的数据
-                //console.log(e.data);
-                putPacket(e.data);
-                //source_buffer.appendBuffer(e.data);
-            }
-        }
-
-        function startup() {
-            ms.addEventListener('sourceopen', opened, false);
-            // get reference to video
-            stream_live = document.getElementById('stream_live');
-            // set mediasource as source of video
-            stream_live.src = window.URL.createObjectURL(ms);
-        }
-        function base64ToArrayBuffer(base64) {
-            var binary_string = window.atob(base64);
-            var len = binary_string.length;
-            var bytes = new Uint8Array(len);
-            for (var i = 0; i < len; i++) {
-                bytes[i] = binary_string.charCodeAt(i);
-            }
-            return bytes.buffer;
+                this.frameQueue.push(e.data);
+                if (!sourcebuffer || sourcebuffer.updating) {
+                    return;
+                }
+                if (this.frameQueue.length === 1) {
+                    sourcebuffer.appendBuffer(this.frameQueue.shift());
+                } else {
+                    let byte_length = 0;
+                    for (const qnode of this.frameQueue) {
+                        byte_length += qnode.byteLength;
+                    }
+                    let mp4buf = new Uint8Array(byte_length);
+                    let offset = 0;
+                    for (const qnode of this.frameQueue) {
+                        let frame = new Uint8Array(qnode);
+                        mp4buf.set(frame, offset);
+                        offset += qnode.byteLength;
+                    }
+                    sourcebuffer.appendBuffer(mp4buf);
+                    this.frameQueue.splice(0, this.frameQueue.length);
+                }
+            }.bind(this);
         }
-        window.onload = function () {
-            startup();
+        JT1078Player.prototype.close = function () {
+            this.ws && this.ws.close();
         }
-
+        document.addEventListener('DOMContentLoaded', function () {
+            var player = new JT1078Player("stream_live", "ws://127.0.0.1:15555/live.mp4?sim=11111111111&channel=1&token=123456");
+            player.open();
+        });
+      
     </script>
 </body>
 </html>
\ No newline at end of file
diff --git a/src/JT1078.Gateway.sln b/src/JT1078.Gateway.sln
index 74d7b8d..64445a3 100644
--- a/src/JT1078.Gateway.sln
+++ b/src/JT1078.Gateway.sln
@@ -20,8 +20,11 @@ EndProject
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{6631FE91-3525-4A84-937B-781C73B343C9}"
 	ProjectSection(SolutionItems) = preProject
 		Info.props = Info.props
+		..\README.md = ..\README.md
 	EndProjectSection
 EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "JT1078.FMp4", "..\..\JT1078\src\JT1078.FMp4\JT1078.FMp4.csproj", "{579E0BB3-98C3-4CC8-8771-DFB7F2B86CC4}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -52,6 +55,10 @@ Global
 		{011934D6-BEE7-4CDA-9F67-4D6D7D672D6C}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{011934D6-BEE7-4CDA-9F67-4D6D7D672D6C}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{011934D6-BEE7-4CDA-9F67-4D6D7D672D6C}.Release|Any CPU.Build.0 = Release|Any CPU
+		{579E0BB3-98C3-4CC8-8771-DFB7F2B86CC4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{579E0BB3-98C3-4CC8-8771-DFB7F2B86CC4}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{579E0BB3-98C3-4CC8-8771-DFB7F2B86CC4}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{579E0BB3-98C3-4CC8-8771-DFB7F2B86CC4}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
diff --git a/src/JT1078.Gateway/JT1078.Gateway.csproj b/src/JT1078.Gateway/JT1078.Gateway.csproj
index 9182d15..dad76f8 100644
--- a/src/JT1078.Gateway/JT1078.Gateway.csproj
+++ b/src/JT1078.Gateway/JT1078.Gateway.csproj
@@ -27,9 +27,9 @@
   </ItemGroup>
 
   <ItemGroup>
-    <PackageReference Include="JT1078.Hls" Version="1.2.0-preview2" />
-    <PackageReference Include="JT1078.FMp4" Version="1.2.0-preview2" />
-    <PackageReference Include="JT1078.Flv" Version="1.2.0-preview2" />
+    <PackageReference Include="JT1078.FMp4" Version="1.2.0-preview6" />
+    <PackageReference Include="JT1078.Hls" Version="1.2.0-preview6" />
+    <PackageReference Include="JT1078.Flv" Version="1.2.0-preview6" />
     <PackageReference Include="Microsoft.Extensions.Caching.Abstractions" Version="6.0.0" />
     <PackageReference Include="System.IO.Pipelines" Version="6.0.1" />
   </ItemGroup>
@@ -40,6 +40,7 @@
    </ItemGroup>
 
   <ItemGroup>
+    <!--<ProjectReference Include="..\..\..\JT1078\src\JT1078.FMp4\JT1078.FMp4.csproj" />-->
     <ProjectReference Include="..\JT1078.Gateway.Abstractions\JT1078.Gateway.Abstractions.csproj" />
   </ItemGroup>
 </Project>