thefoxon/joy4
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add support for VP9 and AV1

Browse Source
This commit is contained in:
Ingo Oppermann 2024-01-03 15:22:18 +01:00
parent 4ee14576ae
commit 1ecb9c2b68
5 changed files with 255 additions and 99 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
codec/av1parser/parser.go Normal file
View File

@ -0,0 +1,31 @@
package av1parser
import (
"github.com/datarhei/joy4/av"
)
type CodecData struct {
Record []byte
}
func (codec CodecData) Type() av.CodecType {
return av.AV1
}
func (codec CodecData) AV1DecoderConfRecordBytes() []byte {
return codec.Record
}
func (codec CodecData) Width() int {
return 0
}
func (codec CodecData) Height() int {
return 0
}
func NewCodecDataFromAV1DecoderConfRecord(record []byte) (self CodecData, err error) {
self.Record = record
return
}

31
codec/vp9parser/parser.go Normal file
View File

@ -0,0 +1,31 @@
package vp9parser
import (
"github.com/datarhei/joy4/av"
)
type CodecData struct {
Record []byte
}
func (codec CodecData) Type() av.CodecType {
return av.VP9
}
func (codec CodecData) VPDecoderConfRecordBytes() []byte {
return codec.Record
}
func (codec CodecData) Width() int {
return 0
}
func (codec CodecData) Height() int {
return 0
}
func NewCodecDataFromVPDecoderConfRecord(record []byte) (self CodecData, err error) {
self.Record = record
return
}

278
format/flv/flv.go
View File

@ -2,7 +2,6 @@ package flv
import (
"bufio"
"encoding/hex"
"fmt"
"io"
@ -10,9 +9,11 @@ import (
"github.com/datarhei/joy4/av/avutil"
"github.com/datarhei/joy4/codec"
"github.com/datarhei/joy4/codec/aacparser"
"github.com/datarhei/joy4/codec/av1parser"
"github.com/datarhei/joy4/codec/fake"
"github.com/datarhei/joy4/codec/h264parser"
"github.com/datarhei/joy4/codec/hevcparser"
"github.com/datarhei/joy4/codec/vp9parser"
"github.com/datarhei/joy4/format/flv/flvio"
"github.com/datarhei/joy4/utils/bits/pio"
)
@ -32,16 +33,27 @@ func NewMetadataByStreams(streams []av.CodecData) (metadata flvio.AMFMap, err er
metadata["videocodecid"] = flvio.VIDEO_H264
case av.HEVC:
metadata["videocodecid"] = flvio.FourCCToFloat(flvio.FOURCC_HEVC)
case av.VP9:
metadata["videocodecid"] = flvio.FourCCToFloat(flvio.FOURCC_VP9)
case av.AV1:
metadata["videocodecid"] = flvio.FourCCToFloat(flvio.FOURCC_AV1)
default:
err = fmt.Errorf("flv: metadata: unsupported video codecType=%v", stream.Type())
return
}
metadata["width"] = stream.Width()
metadata["height"] = stream.Height()
metadata["displayWidth"] = stream.Width()
metadata["displayHeight"] = stream.Height()
width, height := stream.Width(), stream.Height()
if width != 0 {
metadata["width"] = width
metadata["displayWidth"] = width
}
if height != 0 {
metadata["height"] = height
metadata["displayHeight"] = height
}
case typ.IsAudio():
stream := _stream.(av.AudioCodecData)
@ -73,15 +85,15 @@ type Prober struct {
CachedPkts []av.Packet
}
func (self *Prober) CacheTag(_tag flvio.Tag, timestamp int32) {
pkt, _ := self.TagToPacket(_tag, timestamp)
self.CachedPkts = append(self.CachedPkts, pkt)
func (prober *Prober) CacheTag(_tag flvio.Tag, timestamp int32) {
pkt, _ := prober.TagToPacket(_tag, timestamp)
prober.CachedPkts = append(prober.CachedPkts, pkt)
}
func (self *Prober) PushTag(tag flvio.Tag, timestamp int32) (err error) {
self.PushedCount++
func (prober *Prober) PushTag(tag flvio.Tag, timestamp int32) (err error) {
prober.PushedCount++
if self.PushedCount > MaxProbePacketCount {
if prober.PushedCount > MaxProbePacketCount {
err = fmt.Errorf("flv: max probe packet count reached")
return
}
@ -91,38 +103,70 @@ func (self *Prober) PushTag(tag flvio.Tag, timestamp int32) (err error) {
if tag.IsExHeader {
if tag.FourCC == flvio.FOURCC_HEVC {
if tag.PacketType == flvio.PKTTYPE_SEQUENCE_START {
if !self.GotVideo {
if !prober.GotVideo {
var stream hevcparser.CodecData
fmt.Printf("got HEVC sequence start:\n%s\n", hex.Dump(tag.Data))
//fmt.Printf("got HEVC sequence start:\n%s\n", hex.Dump(tag.Data))
if stream, err = hevcparser.NewCodecDataFromHEVCDecoderConfRecord(tag.Data); err != nil {
err = fmt.Errorf("flv: hevc seqhdr invalid: %s", err.Error())
return
}
self.VideoStreamIdx = len(self.Streams)
self.Streams = append(self.Streams, stream)
self.GotVideo = true
prober.VideoStreamIdx = len(prober.Streams)
prober.Streams = append(prober.Streams, stream)
prober.GotVideo = true
}
} else if tag.PacketType == flvio.PKTTYPE_CODED_FRAMES || tag.PacketType == flvio.PKTTYPE_CODED_FRAMESX {
self.CacheTag(tag, timestamp)
prober.CacheTag(tag, timestamp)
}
} else if tag.FourCC == flvio.FOURCC_VP9 {
if tag.PacketType == flvio.PKTTYPE_SEQUENCE_START {
if !prober.GotVideo {
var stream vp9parser.CodecData
//fmt.Printf("got VP9 sequence start:\n%s\n", hex.Dump(tag.Data))
if stream, err = vp9parser.NewCodecDataFromVPDecoderConfRecord(tag.Data); err != nil {
err = fmt.Errorf("flv: vp9 seqhdr invalid: %s", err.Error())
return
}
prober.VideoStreamIdx = len(prober.Streams)
prober.Streams = append(prober.Streams, stream)
prober.GotVideo = true
}
} else if tag.PacketType == flvio.PKTTYPE_CODED_FRAMES || tag.PacketType == flvio.PKTTYPE_CODED_FRAMESX {
prober.CacheTag(tag, timestamp)
}
} else if tag.FourCC == flvio.FOURCC_AV1 {
if tag.PacketType == flvio.PKTTYPE_SEQUENCE_START {
if !prober.GotVideo {
var stream av1parser.CodecData
//fmt.Printf("got AV1 sequence start:\n%s\n", hex.Dump(tag.Data))
if stream, err = av1parser.NewCodecDataFromAV1DecoderConfRecord(tag.Data); err != nil {
err = fmt.Errorf("flv: av1 seqhdr invalid: %s", err.Error())
return
}
prober.VideoStreamIdx = len(prober.Streams)
prober.Streams = append(prober.Streams, stream)
prober.GotVideo = true
}
} else if tag.PacketType == flvio.PKTTYPE_CODED_FRAMES || tag.PacketType == flvio.PKTTYPE_CODED_FRAMESX || tag.PacketType == flvio.PKTTYPE_MPEG2TS_SEQUENCE_START {
prober.CacheTag(tag, timestamp)
}
}
} else {
switch tag.AVCPacketType {
case flvio.AVC_SEQHDR:
if !self.GotVideo {
if !prober.GotVideo {
var stream h264parser.CodecData
fmt.Printf("got H264 sequence start:\n%s\n", hex.Dump(tag.Data))
//fmt.Printf("got H264 sequence start:\n%s\n", hex.Dump(tag.Data))
if stream, err = h264parser.NewCodecDataFromAVCDecoderConfRecord(tag.Data); err != nil {
err = fmt.Errorf("flv: h264 seqhdr invalid: %s", err.Error())
return
}
self.VideoStreamIdx = len(self.Streams)
self.Streams = append(self.Streams, stream)
self.GotVideo = true
prober.VideoStreamIdx = len(prober.Streams)
prober.Streams = append(prober.Streams, stream)
prober.GotVideo = true
}
case flvio.AVC_NALU:
self.CacheTag(tag, timestamp)
prober.CacheTag(tag, timestamp)
}
}
@ -131,42 +175,42 @@ func (self *Prober) PushTag(tag flvio.Tag, timestamp int32) (err error) {
case flvio.SOUND_AAC:
switch tag.AACPacketType {
case flvio.AAC_SEQHDR:
if !self.GotAudio {
if !prober.GotAudio {
var stream aacparser.CodecData
if stream, err = aacparser.NewCodecDataFromMPEG4AudioConfigBytes(tag.Data); err != nil {
err = fmt.Errorf("flv: aac seqhdr invalid")
return
}
self.AudioStreamIdx = len(self.Streams)
self.Streams = append(self.Streams, stream)
self.GotAudio = true
prober.AudioStreamIdx = len(prober.Streams)
prober.Streams = append(prober.Streams, stream)
prober.GotAudio = true
}
case flvio.AAC_RAW:
self.CacheTag(tag, timestamp)
prober.CacheTag(tag, timestamp)
}
case flvio.SOUND_SPEEX:
if !self.GotAudio {
if !prober.GotAudio {
stream := codec.NewSpeexCodecData(16000, tag.ChannelLayout())
self.AudioStreamIdx = len(self.Streams)
self.Streams = append(self.Streams, stream)
self.GotAudio = true
self.CacheTag(tag, timestamp)
prober.AudioStreamIdx = len(prober.Streams)
prober.Streams = append(prober.Streams, stream)
prober.GotAudio = true
prober.CacheTag(tag, timestamp)
}
case flvio.SOUND_NELLYMOSER:
if !self.GotAudio {
if !prober.GotAudio {
stream := fake.CodecData{
CodecType_: av.NELLYMOSER,
SampleRate_: 16000,
SampleFormat_: av.S16,
ChannelLayout_: tag.ChannelLayout(),
}
self.AudioStreamIdx = len(self.Streams)
self.Streams = append(self.Streams, stream)
self.GotAudio = true
self.CacheTag(tag, timestamp)
prober.AudioStreamIdx = len(prober.Streams)
prober.Streams = append(prober.Streams, stream)
prober.GotAudio = true
prober.CacheTag(tag, timestamp)
}
}
@ -175,23 +219,23 @@ func (self *Prober) PushTag(tag flvio.Tag, timestamp int32) (err error) {
return
}
func (self *Prober) Probed() (ok bool) {
if self.HasAudio || self.HasVideo {
if self.HasAudio == self.GotAudio && self.HasVideo == self.GotVideo {
func (prober *Prober) Probed() (ok bool) {
if prober.HasAudio || prober.HasVideo {
if prober.HasAudio == prober.GotAudio && prober.HasVideo == prober.GotVideo {
return true
}
} else {
if self.PushedCount == MaxProbePacketCount {
if prober.PushedCount == MaxProbePacketCount {
return true
}
}
return
}
func (self *Prober) TagToPacket(tag flvio.Tag, timestamp int32) (pkt av.Packet, ok bool) {
func (prober *Prober) TagToPacket(tag flvio.Tag, timestamp int32) (pkt av.Packet, ok bool) {
switch tag.Type {
case flvio.TAG_VIDEO:
pkt.Idx = int8(self.VideoStreamIdx)
pkt.Idx = int8(prober.VideoStreamIdx)
switch tag.PacketType {
case flvio.PKTTYPE_CODED_FRAMES, flvio.PKTTYPE_CODED_FRAMESX:
ok = true
@ -201,7 +245,7 @@ func (self *Prober) TagToPacket(tag flvio.Tag, timestamp int32) (pkt av.Packet,
}
case flvio.TAG_AUDIO:
pkt.Idx = int8(self.AudioStreamIdx)
pkt.Idx = int8(prober.AudioStreamIdx)
switch tag.SoundFormat {
case flvio.SOUND_AAC:
switch tag.AACPacketType {
@ -224,13 +268,13 @@ func (self *Prober) TagToPacket(tag flvio.Tag, timestamp int32) (pkt av.Packet,
return
}
func (self *Prober) Empty() bool {
return len(self.CachedPkts) == 0
func (prober *Prober) Empty() bool {
return len(prober.CachedPkts) == 0
}
func (self *Prober) PopPacket() av.Packet {
pkt := self.CachedPkts[0]
self.CachedPkts = self.CachedPkts[1:]
func (prober *Prober) PopPacket() av.Packet {
pkt := prober.CachedPkts[0]
prober.CachedPkts = prober.CachedPkts[1:]
return pkt
}
@ -245,12 +289,11 @@ func CodecDataToTag(stream av.CodecData) (_tag flvio.Tag, ok bool, err error) {
Data: h264.AVCDecoderConfRecordBytes(),
FrameType: flvio.FRAME_KEY,
}
fmt.Printf("set H264 sequence start:\n%v\n", hex.Dump(h264.AVCDecoderConfRecordBytes()))
//fmt.Printf("set H264 sequence start:\n%v\n", hex.Dump(h264.AVCDecoderConfRecordBytes()))
ok = true
_tag = tag
case av.HEVC:
fmt.Printf("CodecDataToTag for HEVC\n")
hevc := stream.(hevcparser.CodecData)
tag := flvio.Tag{
Type: flvio.TAG_VIDEO,
@ -260,7 +303,35 @@ func CodecDataToTag(stream av.CodecData) (_tag flvio.Tag, ok bool, err error) {
Data: hevc.HEVCDecoderConfRecordBytes(),
FrameType: flvio.FRAME_KEY,
}
fmt.Printf("set HEVC sequence start:\n%v\n", hex.Dump(hevc.HEVCDecoderConfRecordBytes()))
//fmt.Printf("set HEVC sequence start:\n%v\n", hex.Dump(hevc.HEVCDecoderConfRecordBytes()))
ok = true
_tag = tag
case av.VP9:
vp9 := stream.(vp9parser.CodecData)
tag := flvio.Tag{
Type: flvio.TAG_VIDEO,
IsExHeader: true,
PacketType: flvio.PKTTYPE_SEQUENCE_START,
FourCC: flvio.FOURCC_VP9,
Data: vp9.VPDecoderConfRecordBytes(),
FrameType: flvio.FRAME_KEY,
}
//fmt.Printf("set VP9 sequence start:\n%v\n", hex.Dump(vp9.VPDecoderConfRecordBytes()))
ok = true
_tag = tag
case av.AV1:
av1 := stream.(av1parser.CodecData)
tag := flvio.Tag{
Type: flvio.TAG_VIDEO,
IsExHeader: true,
PacketType: flvio.PKTTYPE_SEQUENCE_START,
FourCC: flvio.FOURCC_AV1,
Data: av1.AV1DecoderConfRecordBytes(),
FrameType: flvio.FRAME_KEY,
}
//fmt.Printf("set AV1 sequence start:\n%v\n", hex.Dump(av1.AV1DecoderConfRecordBytes()))
ok = true
_tag = tag
@ -315,7 +386,6 @@ func PacketToTag(pkt av.Packet, stream av.CodecData) (tag flvio.Tag, timestamp i
}
case av.HEVC:
//fmt.Printf("PacketToTag for HEVC\n")
tag = flvio.Tag{
Type: flvio.TAG_VIDEO,
IsExHeader: true,
@ -335,6 +405,38 @@ func PacketToTag(pkt av.Packet, stream av.CodecData) (tag flvio.Tag, timestamp i
tag.FrameType = flvio.FRAME_INTER
}
case av.VP9:
tag = flvio.Tag{
Type: flvio.TAG_VIDEO,
IsExHeader: true,
PacketType: flvio.PKTTYPE_CODED_FRAMES,
CompositionTime: flvio.TimeToTs(pkt.CompositionTime),
FourCC: flvio.FOURCC_VP9,
Data: pkt.Data,
}
if pkt.IsKeyFrame {
tag.FrameType = flvio.FRAME_KEY
} else {
tag.FrameType = flvio.FRAME_INTER
}
case av.AV1:
tag = flvio.Tag{
Type: flvio.TAG_VIDEO,
IsExHeader: true,
PacketType: flvio.PKTTYPE_CODED_FRAMES,
CompositionTime: flvio.TimeToTs(pkt.CompositionTime),
FourCC: flvio.FOURCC_AV1,
Data: pkt.Data,
}
if pkt.IsKeyFrame {
tag.FrameType = flvio.FRAME_KEY
} else {
tag.FrameType = flvio.FRAME_INTER
}
case av.AAC:
tag = flvio.Tag{
Type: flvio.TAG_AUDIO,
@ -398,9 +500,9 @@ func NewMuxer(w io.Writer) *Muxer {
return NewMuxerWriteFlusher(bufio.NewWriterSize(w, pio.RecommendBufioSize))
}
var CodecTypes = []av.CodecType{av.H264, av.AAC, av.SPEEX}
var CodecTypes = []av.CodecType{av.H264, av.HEVC, av.VP9, av.AV1, av.AAC, av.SPEEX}
func (self *Muxer) WriteHeader(streams []av.CodecData) (err error) {
func (muxer *Muxer) WriteHeader(streams []av.CodecData) (err error) {
var flags uint8
for _, stream := range streams {
if stream.Type().IsVideo() {
@ -410,8 +512,8 @@ func (self *Muxer) WriteHeader(streams []av.CodecData) (err error) {
}
}
n := flvio.FillFileHeader(self.b, flags)
if _, err = self.bufw.Write(self.b[:n]); err != nil {
n := flvio.FillFileHeader(muxer.b, flags)
if _, err = muxer.bufw.Write(muxer.b[:n]); err != nil {
return
}
@ -422,28 +524,28 @@ func (self *Muxer) WriteHeader(streams []av.CodecData) (err error) {
return
}
if ok {
if err = flvio.WriteTag(self.bufw, tag, 0, self.b); err != nil {
if err = flvio.WriteTag(muxer.bufw, tag, 0, muxer.b); err != nil {
return
}
}
}
self.streams = streams
muxer.streams = streams
return
}
func (self *Muxer) WritePacket(pkt av.Packet) (err error) {
stream := self.streams[pkt.Idx]
func (muxer *Muxer) WritePacket(pkt av.Packet) (err error) {
stream := muxer.streams[pkt.Idx]
tag, timestamp := PacketToTag(pkt, stream)
if err = flvio.WriteTag(self.bufw, tag, timestamp, self.b); err != nil {
if err = flvio.WriteTag(muxer.bufw, tag, timestamp, muxer.b); err != nil {
return
}
return
}
func (self *Muxer) WriteTrailer() (err error) {
if err = self.bufw.Flush(); err != nil {
func (muxer *Muxer) WriteTrailer() (err error) {
if err = muxer.bufw.Flush(); err != nil {
return
}
return
@ -464,78 +566,76 @@ func NewDemuxer(r io.Reader) *Demuxer {
}
}
func (self *Demuxer) prepare() (err error) {
for self.stage < 2 {
switch self.stage {
func (demuxer *Demuxer) prepare() (err error) {
for demuxer.stage < 2 {
switch demuxer.stage {
case 0:
if _, err = io.ReadFull(self.bufr, self.b[:flvio.FileHeaderLength]); err != nil {
if _, err = io.ReadFull(demuxer.bufr, demuxer.b[:flvio.FileHeaderLength]); err != nil {
return
}
var flags uint8
var skip int
if flags, skip, err = flvio.ParseFileHeader(self.b); err != nil {
if flags, skip, err = flvio.ParseFileHeader(demuxer.b); err != nil {
return
}
if _, err = self.bufr.Discard(skip); err != nil {
if _, err = demuxer.bufr.Discard(skip); err != nil {
return
}
if flags&flvio.FILE_HAS_AUDIO != 0 {
self.prober.HasAudio = true
demuxer.prober.HasAudio = true
}
if flags&flvio.FILE_HAS_VIDEO != 0 {
self.prober.HasVideo = true
demuxer.prober.HasVideo = true
}
self.stage++
demuxer.stage++
case 1:
for !self.prober.Probed() {
for !demuxer.prober.Probed() {
var tag flvio.Tag
var timestamp int32
if tag, timestamp, err = flvio.ReadTag(self.bufr, self.b); err != nil {
if tag, timestamp, err = flvio.ReadTag(demuxer.bufr, demuxer.b); err != nil {
return
}
if err = self.prober.PushTag(tag, timestamp); err != nil {
if err = demuxer.prober.PushTag(tag, timestamp); err != nil {
return
}
}
self.stage++
demuxer.stage++
}
}
return
}
func (self *Demuxer) Streams() (streams []av.CodecData, err error) {
if err = self.prepare(); err != nil {
func (demuxer *Demuxer) Streams() (streams []av.CodecData, err error) {
if err = demuxer.prepare(); err != nil {
return
}
streams = self.prober.Streams
streams = demuxer.prober.Streams
return
}
func (self *Demuxer) ReadPacket() (pkt av.Packet, err error) {
if err = self.prepare(); err != nil {
func (demuxer *Demuxer) ReadPacket() (pkt av.Packet, err error) {
if err = demuxer.prepare(); err != nil {
return
}
if !self.prober.Empty() {
pkt = self.prober.PopPacket()
if !demuxer.prober.Empty() {
pkt = demuxer.prober.PopPacket()
return
}
for {
var tag flvio.Tag
var timestamp int32
if tag, timestamp, err = flvio.ReadTag(self.bufr, self.b); err != nil {
if tag, timestamp, err = flvio.ReadTag(demuxer.bufr, demuxer.b); err != nil {
return
}
var ok bool
if pkt, ok = self.prober.TagToPacket(tag, timestamp); ok {
if pkt, ok = demuxer.prober.TagToPacket(tag, timestamp); ok {
return
}
}
return
}
func Handler(h *avutil.RegisterHandler) {

2
format/flv/flvio/flvio.go
View File

@ -316,7 +316,7 @@ func (t *Tag) videoParseHeader(b []byte) (n int, err error) {
}
}
//fmt.Printf("parseVideoHeader: PacketType: %d\n%s\n", t.PacketType, hex.Dump(b[:n]))
//fmt.Printf("parseVideoHeader: PacketType: %d\n", t.PacketType)
return
}

12
format/rtmp/rtmp.go
View File

@ -78,7 +78,6 @@ func (self *Server) handleConn(conn *Conn) (err error) {
}
if conn.playing {
fmt.Printf("play\n")
if self.HandlePlay != nil {
self.HandlePlay(conn)
}
@ -464,7 +463,7 @@ func (self *Conn) readConnect() (err error) {
return
}
fmt.Printf("readConnect: %+v\n", self.commandobj)
//fmt.Printf("readConnect: %+v\n", self.commandobj)
var ok bool
var _app, _tcurl interface{}
@ -992,17 +991,12 @@ func (self *Conn) WriteHeader(streams []av.CodecData) (err error) {
//metadata = self.GetMetaData()
fmt.Printf("WriteHeader\n")
if metadata == nil {
if metadata, err = flv.NewMetadataByStreams(streams); err != nil {
fmt.Printf("WriteHeader error: %s\n", err.Error())
return
}
}
fmt.Printf("WriteHeader: %#v\n", metadata)
// > onMetaData()
if err = self.writeDataMsg(5, self.avmsgsid, "onMetaData", metadata); err != nil {
return
@ -1558,8 +1552,8 @@ func (self *Conn) handleMsg(timestamp uint32, msgsid uint32, msgtypeid uint8, ms
if metaindex != -1 && metaindex < len(self.datamsgvals) {
self.metadata = self.datamsgvals[metaindex].(flvio.AMFMap)
fmt.Printf("onMetadata: %+v\n", self.metadata)
fmt.Printf("videocodecid: %#08x (%f)\n", int64(self.metadata["videocodecid"].(float64)), self.metadata["videocodecid"].(float64))
//fmt.Printf("onMetadata: %+v\n", self.metadata)
//fmt.Printf("videocodecid: %#08x (%f)\n", int64(self.metadata["videocodecid"].(float64)), self.metadata["videocodecid"].(float64))
}
case msgtypeidVideoMsg: