diff --git a/codec/av1parser/parser.go b/codec/av1parser/parser.go new file mode 100644 index 0000000..7ee0d2c --- /dev/null +++ b/codec/av1parser/parser.go @@ -0,0 +1,31 @@ +package av1parser + +import ( + "github.com/datarhei/joy4/av" +) + +type CodecData struct { + Record []byte +} + +func (codec CodecData) Type() av.CodecType { + return av.AV1 +} + +func (codec CodecData) AV1DecoderConfRecordBytes() []byte { + return codec.Record +} + +func (codec CodecData) Width() int { + return 0 +} + +func (codec CodecData) Height() int { + return 0 +} + +func NewCodecDataFromAV1DecoderConfRecord(record []byte) (self CodecData, err error) { + self.Record = record + + return +} diff --git a/codec/vp9parser/parser.go b/codec/vp9parser/parser.go new file mode 100644 index 0000000..d5bbaa6 --- /dev/null +++ b/codec/vp9parser/parser.go @@ -0,0 +1,31 @@ +package vp9parser + +import ( + "github.com/datarhei/joy4/av" +) + +type CodecData struct { + Record []byte +} + +func (codec CodecData) Type() av.CodecType { + return av.VP9 +} + +func (codec CodecData) VPDecoderConfRecordBytes() []byte { + return codec.Record +} + +func (codec CodecData) Width() int { + return 0 +} + +func (codec CodecData) Height() int { + return 0 +} + +func NewCodecDataFromVPDecoderConfRecord(record []byte) (self CodecData, err error) { + self.Record = record + + return +} diff --git a/format/flv/flv.go b/format/flv/flv.go index 5bac86d..26fc3ab 100644 --- a/format/flv/flv.go +++ b/format/flv/flv.go @@ -2,7 +2,6 @@ package flv import ( "bufio" - "encoding/hex" "fmt" "io" @@ -10,9 +9,11 @@ import ( "github.com/datarhei/joy4/av/avutil" "github.com/datarhei/joy4/codec" "github.com/datarhei/joy4/codec/aacparser" + "github.com/datarhei/joy4/codec/av1parser" "github.com/datarhei/joy4/codec/fake" "github.com/datarhei/joy4/codec/h264parser" "github.com/datarhei/joy4/codec/hevcparser" + "github.com/datarhei/joy4/codec/vp9parser" "github.com/datarhei/joy4/format/flv/flvio" "github.com/datarhei/joy4/utils/bits/pio" ) @@ -32,16 +33,27 @@ func NewMetadataByStreams(streams []av.CodecData) (metadata flvio.AMFMap, err er metadata["videocodecid"] = flvio.VIDEO_H264 case av.HEVC: metadata["videocodecid"] = flvio.FourCCToFloat(flvio.FOURCC_HEVC) + case av.VP9: + metadata["videocodecid"] = flvio.FourCCToFloat(flvio.FOURCC_VP9) + case av.AV1: + metadata["videocodecid"] = flvio.FourCCToFloat(flvio.FOURCC_AV1) default: err = fmt.Errorf("flv: metadata: unsupported video codecType=%v", stream.Type()) return } - metadata["width"] = stream.Width() - metadata["height"] = stream.Height() - metadata["displayWidth"] = stream.Width() - metadata["displayHeight"] = stream.Height() + width, height := stream.Width(), stream.Height() + + if width != 0 { + metadata["width"] = width + metadata["displayWidth"] = width + } + + if height != 0 { + metadata["height"] = height + metadata["displayHeight"] = height + } case typ.IsAudio(): stream := _stream.(av.AudioCodecData) @@ -73,15 +85,15 @@ type Prober struct { CachedPkts []av.Packet } -func (self *Prober) CacheTag(_tag flvio.Tag, timestamp int32) { - pkt, _ := self.TagToPacket(_tag, timestamp) - self.CachedPkts = append(self.CachedPkts, pkt) +func (prober *Prober) CacheTag(_tag flvio.Tag, timestamp int32) { + pkt, _ := prober.TagToPacket(_tag, timestamp) + prober.CachedPkts = append(prober.CachedPkts, pkt) } -func (self *Prober) PushTag(tag flvio.Tag, timestamp int32) (err error) { - self.PushedCount++ +func (prober *Prober) PushTag(tag flvio.Tag, timestamp int32) (err error) { + prober.PushedCount++ - if self.PushedCount > MaxProbePacketCount { + if prober.PushedCount > MaxProbePacketCount { err = fmt.Errorf("flv: max probe packet count reached") return } @@ -91,38 +103,70 @@ func (self *Prober) PushTag(tag flvio.Tag, timestamp int32) (err error) { if tag.IsExHeader { if tag.FourCC == flvio.FOURCC_HEVC { if tag.PacketType == flvio.PKTTYPE_SEQUENCE_START { - if !self.GotVideo { + if !prober.GotVideo { var stream hevcparser.CodecData - fmt.Printf("got HEVC sequence start:\n%s\n", hex.Dump(tag.Data)) + //fmt.Printf("got HEVC sequence start:\n%s\n", hex.Dump(tag.Data)) if stream, err = hevcparser.NewCodecDataFromHEVCDecoderConfRecord(tag.Data); err != nil { err = fmt.Errorf("flv: hevc seqhdr invalid: %s", err.Error()) return } - self.VideoStreamIdx = len(self.Streams) - self.Streams = append(self.Streams, stream) - self.GotVideo = true + prober.VideoStreamIdx = len(prober.Streams) + prober.Streams = append(prober.Streams, stream) + prober.GotVideo = true } } else if tag.PacketType == flvio.PKTTYPE_CODED_FRAMES || tag.PacketType == flvio.PKTTYPE_CODED_FRAMESX { - self.CacheTag(tag, timestamp) + prober.CacheTag(tag, timestamp) + } + } else if tag.FourCC == flvio.FOURCC_VP9 { + if tag.PacketType == flvio.PKTTYPE_SEQUENCE_START { + if !prober.GotVideo { + var stream vp9parser.CodecData + //fmt.Printf("got VP9 sequence start:\n%s\n", hex.Dump(tag.Data)) + if stream, err = vp9parser.NewCodecDataFromVPDecoderConfRecord(tag.Data); err != nil { + err = fmt.Errorf("flv: vp9 seqhdr invalid: %s", err.Error()) + return + } + prober.VideoStreamIdx = len(prober.Streams) + prober.Streams = append(prober.Streams, stream) + prober.GotVideo = true + } + } else if tag.PacketType == flvio.PKTTYPE_CODED_FRAMES || tag.PacketType == flvio.PKTTYPE_CODED_FRAMESX { + prober.CacheTag(tag, timestamp) + } + } else if tag.FourCC == flvio.FOURCC_AV1 { + if tag.PacketType == flvio.PKTTYPE_SEQUENCE_START { + if !prober.GotVideo { + var stream av1parser.CodecData + //fmt.Printf("got AV1 sequence start:\n%s\n", hex.Dump(tag.Data)) + if stream, err = av1parser.NewCodecDataFromAV1DecoderConfRecord(tag.Data); err != nil { + err = fmt.Errorf("flv: av1 seqhdr invalid: %s", err.Error()) + return + } + prober.VideoStreamIdx = len(prober.Streams) + prober.Streams = append(prober.Streams, stream) + prober.GotVideo = true + } + } else if tag.PacketType == flvio.PKTTYPE_CODED_FRAMES || tag.PacketType == flvio.PKTTYPE_CODED_FRAMESX || tag.PacketType == flvio.PKTTYPE_MPEG2TS_SEQUENCE_START { + prober.CacheTag(tag, timestamp) } } } else { switch tag.AVCPacketType { case flvio.AVC_SEQHDR: - if !self.GotVideo { + if !prober.GotVideo { var stream h264parser.CodecData - fmt.Printf("got H264 sequence start:\n%s\n", hex.Dump(tag.Data)) + //fmt.Printf("got H264 sequence start:\n%s\n", hex.Dump(tag.Data)) if stream, err = h264parser.NewCodecDataFromAVCDecoderConfRecord(tag.Data); err != nil { err = fmt.Errorf("flv: h264 seqhdr invalid: %s", err.Error()) return } - self.VideoStreamIdx = len(self.Streams) - self.Streams = append(self.Streams, stream) - self.GotVideo = true + prober.VideoStreamIdx = len(prober.Streams) + prober.Streams = append(prober.Streams, stream) + prober.GotVideo = true } case flvio.AVC_NALU: - self.CacheTag(tag, timestamp) + prober.CacheTag(tag, timestamp) } } @@ -131,42 +175,42 @@ func (self *Prober) PushTag(tag flvio.Tag, timestamp int32) (err error) { case flvio.SOUND_AAC: switch tag.AACPacketType { case flvio.AAC_SEQHDR: - if !self.GotAudio { + if !prober.GotAudio { var stream aacparser.CodecData if stream, err = aacparser.NewCodecDataFromMPEG4AudioConfigBytes(tag.Data); err != nil { err = fmt.Errorf("flv: aac seqhdr invalid") return } - self.AudioStreamIdx = len(self.Streams) - self.Streams = append(self.Streams, stream) - self.GotAudio = true + prober.AudioStreamIdx = len(prober.Streams) + prober.Streams = append(prober.Streams, stream) + prober.GotAudio = true } case flvio.AAC_RAW: - self.CacheTag(tag, timestamp) + prober.CacheTag(tag, timestamp) } case flvio.SOUND_SPEEX: - if !self.GotAudio { + if !prober.GotAudio { stream := codec.NewSpeexCodecData(16000, tag.ChannelLayout()) - self.AudioStreamIdx = len(self.Streams) - self.Streams = append(self.Streams, stream) - self.GotAudio = true - self.CacheTag(tag, timestamp) + prober.AudioStreamIdx = len(prober.Streams) + prober.Streams = append(prober.Streams, stream) + prober.GotAudio = true + prober.CacheTag(tag, timestamp) } case flvio.SOUND_NELLYMOSER: - if !self.GotAudio { + if !prober.GotAudio { stream := fake.CodecData{ CodecType_: av.NELLYMOSER, SampleRate_: 16000, SampleFormat_: av.S16, ChannelLayout_: tag.ChannelLayout(), } - self.AudioStreamIdx = len(self.Streams) - self.Streams = append(self.Streams, stream) - self.GotAudio = true - self.CacheTag(tag, timestamp) + prober.AudioStreamIdx = len(prober.Streams) + prober.Streams = append(prober.Streams, stream) + prober.GotAudio = true + prober.CacheTag(tag, timestamp) } } @@ -175,23 +219,23 @@ func (self *Prober) PushTag(tag flvio.Tag, timestamp int32) (err error) { return } -func (self *Prober) Probed() (ok bool) { - if self.HasAudio || self.HasVideo { - if self.HasAudio == self.GotAudio && self.HasVideo == self.GotVideo { +func (prober *Prober) Probed() (ok bool) { + if prober.HasAudio || prober.HasVideo { + if prober.HasAudio == prober.GotAudio && prober.HasVideo == prober.GotVideo { return true } } else { - if self.PushedCount == MaxProbePacketCount { + if prober.PushedCount == MaxProbePacketCount { return true } } return } -func (self *Prober) TagToPacket(tag flvio.Tag, timestamp int32) (pkt av.Packet, ok bool) { +func (prober *Prober) TagToPacket(tag flvio.Tag, timestamp int32) (pkt av.Packet, ok bool) { switch tag.Type { case flvio.TAG_VIDEO: - pkt.Idx = int8(self.VideoStreamIdx) + pkt.Idx = int8(prober.VideoStreamIdx) switch tag.PacketType { case flvio.PKTTYPE_CODED_FRAMES, flvio.PKTTYPE_CODED_FRAMESX: ok = true @@ -201,7 +245,7 @@ func (self *Prober) TagToPacket(tag flvio.Tag, timestamp int32) (pkt av.Packet, } case flvio.TAG_AUDIO: - pkt.Idx = int8(self.AudioStreamIdx) + pkt.Idx = int8(prober.AudioStreamIdx) switch tag.SoundFormat { case flvio.SOUND_AAC: switch tag.AACPacketType { @@ -224,13 +268,13 @@ func (self *Prober) TagToPacket(tag flvio.Tag, timestamp int32) (pkt av.Packet, return } -func (self *Prober) Empty() bool { - return len(self.CachedPkts) == 0 +func (prober *Prober) Empty() bool { + return len(prober.CachedPkts) == 0 } -func (self *Prober) PopPacket() av.Packet { - pkt := self.CachedPkts[0] - self.CachedPkts = self.CachedPkts[1:] +func (prober *Prober) PopPacket() av.Packet { + pkt := prober.CachedPkts[0] + prober.CachedPkts = prober.CachedPkts[1:] return pkt } @@ -245,12 +289,11 @@ func CodecDataToTag(stream av.CodecData) (_tag flvio.Tag, ok bool, err error) { Data: h264.AVCDecoderConfRecordBytes(), FrameType: flvio.FRAME_KEY, } - fmt.Printf("set H264 sequence start:\n%v\n", hex.Dump(h264.AVCDecoderConfRecordBytes())) + //fmt.Printf("set H264 sequence start:\n%v\n", hex.Dump(h264.AVCDecoderConfRecordBytes())) ok = true _tag = tag case av.HEVC: - fmt.Printf("CodecDataToTag for HEVC\n") hevc := stream.(hevcparser.CodecData) tag := flvio.Tag{ Type: flvio.TAG_VIDEO, @@ -260,7 +303,35 @@ func CodecDataToTag(stream av.CodecData) (_tag flvio.Tag, ok bool, err error) { Data: hevc.HEVCDecoderConfRecordBytes(), FrameType: flvio.FRAME_KEY, } - fmt.Printf("set HEVC sequence start:\n%v\n", hex.Dump(hevc.HEVCDecoderConfRecordBytes())) + //fmt.Printf("set HEVC sequence start:\n%v\n", hex.Dump(hevc.HEVCDecoderConfRecordBytes())) + ok = true + _tag = tag + + case av.VP9: + vp9 := stream.(vp9parser.CodecData) + tag := flvio.Tag{ + Type: flvio.TAG_VIDEO, + IsExHeader: true, + PacketType: flvio.PKTTYPE_SEQUENCE_START, + FourCC: flvio.FOURCC_VP9, + Data: vp9.VPDecoderConfRecordBytes(), + FrameType: flvio.FRAME_KEY, + } + //fmt.Printf("set VP9 sequence start:\n%v\n", hex.Dump(vp9.VPDecoderConfRecordBytes())) + ok = true + _tag = tag + + case av.AV1: + av1 := stream.(av1parser.CodecData) + tag := flvio.Tag{ + Type: flvio.TAG_VIDEO, + IsExHeader: true, + PacketType: flvio.PKTTYPE_SEQUENCE_START, + FourCC: flvio.FOURCC_AV1, + Data: av1.AV1DecoderConfRecordBytes(), + FrameType: flvio.FRAME_KEY, + } + //fmt.Printf("set AV1 sequence start:\n%v\n", hex.Dump(av1.AV1DecoderConfRecordBytes())) ok = true _tag = tag @@ -315,7 +386,6 @@ func PacketToTag(pkt av.Packet, stream av.CodecData) (tag flvio.Tag, timestamp i } case av.HEVC: - //fmt.Printf("PacketToTag for HEVC\n") tag = flvio.Tag{ Type: flvio.TAG_VIDEO, IsExHeader: true, @@ -335,6 +405,38 @@ func PacketToTag(pkt av.Packet, stream av.CodecData) (tag flvio.Tag, timestamp i tag.FrameType = flvio.FRAME_INTER } + case av.VP9: + tag = flvio.Tag{ + Type: flvio.TAG_VIDEO, + IsExHeader: true, + PacketType: flvio.PKTTYPE_CODED_FRAMES, + CompositionTime: flvio.TimeToTs(pkt.CompositionTime), + FourCC: flvio.FOURCC_VP9, + Data: pkt.Data, + } + + if pkt.IsKeyFrame { + tag.FrameType = flvio.FRAME_KEY + } else { + tag.FrameType = flvio.FRAME_INTER + } + + case av.AV1: + tag = flvio.Tag{ + Type: flvio.TAG_VIDEO, + IsExHeader: true, + PacketType: flvio.PKTTYPE_CODED_FRAMES, + CompositionTime: flvio.TimeToTs(pkt.CompositionTime), + FourCC: flvio.FOURCC_AV1, + Data: pkt.Data, + } + + if pkt.IsKeyFrame { + tag.FrameType = flvio.FRAME_KEY + } else { + tag.FrameType = flvio.FRAME_INTER + } + case av.AAC: tag = flvio.Tag{ Type: flvio.TAG_AUDIO, @@ -398,9 +500,9 @@ func NewMuxer(w io.Writer) *Muxer { return NewMuxerWriteFlusher(bufio.NewWriterSize(w, pio.RecommendBufioSize)) } -var CodecTypes = []av.CodecType{av.H264, av.AAC, av.SPEEX} +var CodecTypes = []av.CodecType{av.H264, av.HEVC, av.VP9, av.AV1, av.AAC, av.SPEEX} -func (self *Muxer) WriteHeader(streams []av.CodecData) (err error) { +func (muxer *Muxer) WriteHeader(streams []av.CodecData) (err error) { var flags uint8 for _, stream := range streams { if stream.Type().IsVideo() { @@ -410,8 +512,8 @@ func (self *Muxer) WriteHeader(streams []av.CodecData) (err error) { } } - n := flvio.FillFileHeader(self.b, flags) - if _, err = self.bufw.Write(self.b[:n]); err != nil { + n := flvio.FillFileHeader(muxer.b, flags) + if _, err = muxer.bufw.Write(muxer.b[:n]); err != nil { return } @@ -422,28 +524,28 @@ func (self *Muxer) WriteHeader(streams []av.CodecData) (err error) { return } if ok { - if err = flvio.WriteTag(self.bufw, tag, 0, self.b); err != nil { + if err = flvio.WriteTag(muxer.bufw, tag, 0, muxer.b); err != nil { return } } } - self.streams = streams + muxer.streams = streams return } -func (self *Muxer) WritePacket(pkt av.Packet) (err error) { - stream := self.streams[pkt.Idx] +func (muxer *Muxer) WritePacket(pkt av.Packet) (err error) { + stream := muxer.streams[pkt.Idx] tag, timestamp := PacketToTag(pkt, stream) - if err = flvio.WriteTag(self.bufw, tag, timestamp, self.b); err != nil { + if err = flvio.WriteTag(muxer.bufw, tag, timestamp, muxer.b); err != nil { return } return } -func (self *Muxer) WriteTrailer() (err error) { - if err = self.bufw.Flush(); err != nil { +func (muxer *Muxer) WriteTrailer() (err error) { + if err = muxer.bufw.Flush(); err != nil { return } return @@ -464,78 +566,76 @@ func NewDemuxer(r io.Reader) *Demuxer { } } -func (self *Demuxer) prepare() (err error) { - for self.stage < 2 { - switch self.stage { +func (demuxer *Demuxer) prepare() (err error) { + for demuxer.stage < 2 { + switch demuxer.stage { case 0: - if _, err = io.ReadFull(self.bufr, self.b[:flvio.FileHeaderLength]); err != nil { + if _, err = io.ReadFull(demuxer.bufr, demuxer.b[:flvio.FileHeaderLength]); err != nil { return } var flags uint8 var skip int - if flags, skip, err = flvio.ParseFileHeader(self.b); err != nil { + if flags, skip, err = flvio.ParseFileHeader(demuxer.b); err != nil { return } - if _, err = self.bufr.Discard(skip); err != nil { + if _, err = demuxer.bufr.Discard(skip); err != nil { return } if flags&flvio.FILE_HAS_AUDIO != 0 { - self.prober.HasAudio = true + demuxer.prober.HasAudio = true } if flags&flvio.FILE_HAS_VIDEO != 0 { - self.prober.HasVideo = true + demuxer.prober.HasVideo = true } - self.stage++ + demuxer.stage++ case 1: - for !self.prober.Probed() { + for !demuxer.prober.Probed() { var tag flvio.Tag var timestamp int32 - if tag, timestamp, err = flvio.ReadTag(self.bufr, self.b); err != nil { + if tag, timestamp, err = flvio.ReadTag(demuxer.bufr, demuxer.b); err != nil { return } - if err = self.prober.PushTag(tag, timestamp); err != nil { + if err = demuxer.prober.PushTag(tag, timestamp); err != nil { return } } - self.stage++ + demuxer.stage++ } } return } -func (self *Demuxer) Streams() (streams []av.CodecData, err error) { - if err = self.prepare(); err != nil { +func (demuxer *Demuxer) Streams() (streams []av.CodecData, err error) { + if err = demuxer.prepare(); err != nil { return } - streams = self.prober.Streams + streams = demuxer.prober.Streams return } -func (self *Demuxer) ReadPacket() (pkt av.Packet, err error) { - if err = self.prepare(); err != nil { +func (demuxer *Demuxer) ReadPacket() (pkt av.Packet, err error) { + if err = demuxer.prepare(); err != nil { return } - if !self.prober.Empty() { - pkt = self.prober.PopPacket() + if !demuxer.prober.Empty() { + pkt = demuxer.prober.PopPacket() return } for { var tag flvio.Tag var timestamp int32 - if tag, timestamp, err = flvio.ReadTag(self.bufr, self.b); err != nil { + if tag, timestamp, err = flvio.ReadTag(demuxer.bufr, demuxer.b); err != nil { return } var ok bool - if pkt, ok = self.prober.TagToPacket(tag, timestamp); ok { + if pkt, ok = demuxer.prober.TagToPacket(tag, timestamp); ok { return } } - - return } func Handler(h *avutil.RegisterHandler) { diff --git a/format/flv/flvio/flvio.go b/format/flv/flvio/flvio.go index 9b69085..8c0a57e 100644 --- a/format/flv/flvio/flvio.go +++ b/format/flv/flvio/flvio.go @@ -316,7 +316,7 @@ func (t *Tag) videoParseHeader(b []byte) (n int, err error) { } } - //fmt.Printf("parseVideoHeader: PacketType: %d\n%s\n", t.PacketType, hex.Dump(b[:n])) + //fmt.Printf("parseVideoHeader: PacketType: %d\n", t.PacketType) return } diff --git a/format/rtmp/rtmp.go b/format/rtmp/rtmp.go index 9b8a93f..6a0e1f1 100644 --- a/format/rtmp/rtmp.go +++ b/format/rtmp/rtmp.go @@ -78,7 +78,6 @@ func (self *Server) handleConn(conn *Conn) (err error) { } if conn.playing { - fmt.Printf("play\n") if self.HandlePlay != nil { self.HandlePlay(conn) } @@ -464,7 +463,7 @@ func (self *Conn) readConnect() (err error) { return } - fmt.Printf("readConnect: %+v\n", self.commandobj) + //fmt.Printf("readConnect: %+v\n", self.commandobj) var ok bool var _app, _tcurl interface{} @@ -992,17 +991,12 @@ func (self *Conn) WriteHeader(streams []av.CodecData) (err error) { //metadata = self.GetMetaData() - fmt.Printf("WriteHeader\n") - if metadata == nil { if metadata, err = flv.NewMetadataByStreams(streams); err != nil { - fmt.Printf("WriteHeader error: %s\n", err.Error()) return } } - fmt.Printf("WriteHeader: %#v\n", metadata) - // > onMetaData() if err = self.writeDataMsg(5, self.avmsgsid, "onMetaData", metadata); err != nil { return @@ -1558,8 +1552,8 @@ func (self *Conn) handleMsg(timestamp uint32, msgsid uint32, msgtypeid uint8, ms if metaindex != -1 && metaindex < len(self.datamsgvals) { self.metadata = self.datamsgvals[metaindex].(flvio.AMFMap) - fmt.Printf("onMetadata: %+v\n", self.metadata) - fmt.Printf("videocodecid: %#08x (%f)\n", int64(self.metadata["videocodecid"].(float64)), self.metadata["videocodecid"].(float64)) + //fmt.Printf("onMetadata: %+v\n", self.metadata) + //fmt.Printf("videocodecid: %#08x (%f)\n", int64(self.metadata["videocodecid"].(float64)), self.metadata["videocodecid"].(float64)) } case msgtypeidVideoMsg: