[libav-devel] [PATCH] flv: support stream text data as onTextData

Martin Storsjö martin at martin.st
Sat May 12 21:35:43 CEST 2012


On Wed, 9 May 2012, Luca Barbato wrote:

> Adobe specifies onTextData as the standard message to use to deliver
> text information.
> ---
> libavformat/flvdec.c |  102 ++++++++++++++++++++++++++++++++++++++++++-------
> libavformat/flvenc.c |   77 ++++++++++++++++++++++++++++++++-----
> 2 files changed, 153 insertions(+), 26 deletions(-)
>
> diff --git a/libavformat/flvdec.c b/libavformat/flvdec.c
> index 7c4b792..d2e9fdd 100644
> --- a/libavformat/flvdec.c
> +++ b/libavformat/flvdec.c
> @@ -66,6 +66,17 @@ static int flv_probe(AVProbeData *p)
>     return 0;
> }
>
> +static AVStream *create_stream(AVFormatContext *s, int tag, int codec_type)
> +{
> +    AVStream *st = avformat_new_stream(s, NULL);
> +    if (!st)
> +        return NULL;
> +    st->id = tag;
> +    st->codec->codec_type = codec_type;
> +    avpriv_set_pts_info(st, 32, 1, 1000); /* 32 bit pts in ms */
> +    return st;
> +}
> +
> static void flv_set_audio_codec(AVFormatContext *s, AVStream *astream, AVCodecContext *acodec, int flv_codecid) {
>     switch(flv_codecid) {
>         //no distinction between S16 and S8 PCM codec flags
> @@ -302,6 +313,12 @@ static int amf_parse_object(AVFormatContext *s, AVStream *astream, AVStream *vst
>                 vcodec->bit_rate = num_val * 1024.0;
>             else if (!strcmp(key, "audiodatarate") && acodec && 0 <= (int)(num_val * 1024.0))
>                 acodec->bit_rate = num_val * 1024.0;
> +            else if (!strcmp(key, "datastream")) {
> +                AVStream *st = create_stream(s, 2, AVMEDIA_TYPE_DATA);
> +                if (!st)
> +                    return AVERROR(ENOMEM);
> +                st->codec->codec_id = CODEC_ID_TEXT;
> +            }
>         }
>
>         if (!strcmp(key, "duration")        ||
> @@ -344,7 +361,14 @@ static int flv_read_metabody(AVFormatContext *s, int64_t next_pos) {
>
>     //first object needs to be "onMetaData" string
>     type = avio_r8(ioc);
> -    if(type != AMF_DATA_TYPE_STRING || amf_get_string(ioc, buffer, sizeof(buffer)) < 0 || strcmp(buffer, "onMetaData"))
> +    if (type != AMF_DATA_TYPE_STRING ||
> +        amf_get_string(ioc, buffer, sizeof(buffer)) < 0)
> +        return -1;
> +
> +    if (!strcmp(buffer, "onTextData"))
> +        return 1;
> +
> +    if (strcmp(buffer, "onMetaData"))
>         return -1;
>
>     //find the streams now so that amf_parse_object doesn't need to do the lookup every time it is called.
> @@ -361,16 +385,6 @@ static int flv_read_metabody(AVFormatContext *s, int64_t next_pos) {
>     return 0;
> }
>
> -static AVStream *create_stream(AVFormatContext *s, int is_audio){
> -    AVStream *st = avformat_new_stream(s, NULL);
> -    if (!st)
> -        return NULL;
> -    st->id = is_audio;
> -    st->codec->codec_type = is_audio ? AVMEDIA_TYPE_AUDIO : AVMEDIA_TYPE_VIDEO;
> -    avpriv_set_pts_info(st, 32, 1, 1000); /* 32 bit pts in ms */
> -    return st;
> -}
> -
> static int flv_read_header(AVFormatContext *s)
> {
>     int offset, flags;
> @@ -389,11 +403,11 @@ static int flv_read_header(AVFormatContext *s)
>         s->ctx_flags |= AVFMTCTX_NOHEADER;
>
>     if(flags & FLV_HEADER_FLAG_HASVIDEO){
> -        if(!create_stream(s, 0))
> +        if(!create_stream(s, 0, AVMEDIA_TYPE_VIDEO))
>             return AVERROR(ENOMEM);
>     }
>     if(flags & FLV_HEADER_FLAG_HASAUDIO){
> -        if(!create_stream(s, 1))
> +        if(!create_stream(s, 1, AVMEDIA_TYPE_AUDIO))
>             return AVERROR(ENOMEM);
>     }
>
> @@ -453,6 +467,61 @@ static void clear_index_entries(AVFormatContext *s, int64_t pos)
>     }
> }
>
> +
> +static int flv_data_packet(AVFormatContext *s, AVPacket *pkt,
> +                           int64_t dts, int64_t next)
> +{
> +    AVIOContext *pb = s->pb;
> +    AVStream *st = NULL;
> +    AMFDataType type;
> +    char buf[20];
> +    int codec_id;
> +    int length;
> +    int ret = AVERROR_INVALIDDATA, i;
> +
> +    type = avio_r8(pb);
> +    if (type == AMF_DATA_TYPE_MIXEDARRAY)
> +        avio_seek(pb, 4, SEEK_CUR);
> +    else if (type != AMF_DATA_TYPE_OBJECT)
> +        goto out;
> +    amf_get_string(pb, buf, sizeof(buf));
> +    if (strcmp(buf,"type") || avio_r8(pb) != AMF_DATA_TYPE_STRING)
> +        goto out;

The strcmps could use some space between the parameters

> +    amf_get_string(pb, buf, sizeof(buf));
> +    //FIXME parse it as codec_id
> +    amf_get_string(pb, buf, sizeof(buf));
> +    if (strcmp(buf,"text") || avio_r8(pb) != AMF_DATA_TYPE_STRING)
> +        goto out;
> +    length = avio_rb16(pb);

Is this "type => 'foo', text => 'bar'" structure standardized anywhere or 
used by any other existing application?

> +    ret = av_get_packet(s->pb, pkt, length);
> +    if (ret < 0) {
> +        ret = AVERROR(EIO);
> +        goto out;
> +    }
> +
> +    for(i=0;i<s->nb_streams;i++) {

Spacing please

> +        st = s->streams[i];
> +        if (st->id == 2) //XXX

I agree with Anton, this isn't really nice. Some define for the track ids 
for audio/video/data might be useful.

> +            break;
> +    }
> +    if (st->id != 2) {
> +        st = create_stream(s, 2, AVMEDIA_TYPE_DATA);
> +        if (!st)
> +            goto out;
> +        st->codec->codec_id = CODEC_ID_TEXT;
> +    }
> +
> +    pkt->size = ret;
> +    pkt->dts = dts;
> +    pkt->pts = dts;
> +    pkt->stream_index = st->index;
> +    pkt->flags |= AV_PKT_FLAG_KEY;
> +
> +    avio_seek(s->pb, next + 4, SEEK_SET);
> +out:
> +    return ret;
> +}
> +
> static int flv_read_packet(AVFormatContext *s, AVPacket *pkt)
> {
>     FLVContext *flv = s->priv_data;
> @@ -507,7 +576,9 @@ static int flv_read_packet(AVFormatContext *s, AVPacket *pkt)
>             goto skip;
>     } else {
>         if (type == FLV_TAG_TYPE_META && size > 13+1+4)
> -            flv_read_metabody(s, next);
> +            if (flv_read_metabody(s, next) > 0) {
> +                return flv_data_packet(s, pkt, dts, next);
> +            }
>         else /* skip packet */
>             av_log(s, AV_LOG_DEBUG, "skipping flv packet: type %d, size %d, flags %d\n", type, size, flags);
>     skip:
> @@ -527,7 +598,8 @@ static int flv_read_packet(AVFormatContext *s, AVPacket *pkt)
>     }
>     if(i == s->nb_streams){
>         av_log(s, AV_LOG_ERROR, "invalid stream\n");
> -        st= create_stream(s, is_audio);
> +        st = create_stream(s, is_audio,
> +             is_audio ? AVMEDIA_TYPE_AUDIO : AVMEDIA_TYPE_VIDEO);
>         s->ctx_flags &= ~AVFMTCTX_NOHEADER;
>     }
>     av_dlog(s, "%d %X %d \n", is_audio, flags, st->discard);
> diff --git a/libavformat/flvenc.c b/libavformat/flvenc.c
> index 19169ad..19f1439 100644
> --- a/libavformat/flvenc.c
> +++ b/libavformat/flvenc.c
> @@ -178,7 +178,7 @@ static int flv_write_header(AVFormatContext *s)
> {
>     AVIOContext *pb = s->pb;
>     FLVContext *flv = s->priv_data;
> -    AVCodecContext *audio_enc = NULL, *video_enc = NULL;
> +    AVCodecContext *audio_enc = NULL, *video_enc = NULL, *data_enc = NULL;
>     int i, metadata_count = 0;
>     double framerate = 0.0;
>     int64_t metadata_size_pos, data_size, metadata_count_pos;
> @@ -187,7 +187,8 @@ static int flv_write_header(AVFormatContext *s)
>     for(i=0; i<s->nb_streams; i++){
>         AVCodecContext *enc = s->streams[i]->codec;
>         FLVStreamContext *sc;
> -        if (enc->codec_type == AVMEDIA_TYPE_VIDEO) {
> +        switch (enc->codec_type) {
> +        case AVMEDIA_TYPE_VIDEO:
>             if (s->streams[i]->r_frame_rate.den && s->streams[i]->r_frame_rate.num) {
>                 framerate = av_q2d(s->streams[i]->r_frame_rate);
>             } else {
> @@ -196,12 +197,25 @@ static int flv_write_header(AVFormatContext *s)
>             video_enc = enc;
>             if(enc->codec_tag == 0) {
>                 av_log(s, AV_LOG_ERROR, "video codec not compatible with flv\n");
> -                return -1;
> +                return AVERROR_INVALIDDATA;

Unrelated, please split it to a separate patch

>             }
> -        } else {
> +        break;
> +        case AVMEDIA_TYPE_AUDIO:
>             audio_enc = enc;
>             if (get_audio_flags(s, enc) < 0)
> -                return -1;
> +                return AVERROR_INVALIDDATA;
> +        break;
> +        case AVMEDIA_TYPE_DATA:
> +            if (enc->codec_id != CODEC_ID_TEXT) {
> +                    av_log(s, AV_LOG_ERROR,
> +                                "codec not compatible with flv\n");
> +                    return AVERROR_INVALIDDATA;

Indentation off

> +            }
> +            data_enc = enc;
> +        break;
> +        default:
> +            av_log(s, AV_LOG_ERROR, "codec not compatible with flv\n");
> +            return -1;
>         }
>         avpriv_set_pts_info(s->streams[i], 32, 1, 1000); /* 32 bit pts in ms */
>
> @@ -247,7 +261,11 @@ static int flv_write_header(AVFormatContext *s)
>     /* mixed array (hash) with size and string/type/data tuples */
>     avio_w8(pb, AMF_DATA_TYPE_MIXEDARRAY);
>     metadata_count_pos = avio_tell(pb);
> -    metadata_count = 5*!!video_enc + 5*!!audio_enc + 2; // +2 for duration and file size
> +    metadata_count = 5 * !!video_enc +
> +                     5 * !!audio_enc +
> +                     1 * !!data_enc +
> +                     2; // +2 for duration and file size
> +
>     avio_wb32(pb, metadata_count);
>
>     put_amf_string(pb, "duration");
> @@ -288,6 +306,11 @@ static int flv_write_header(AVFormatContext *s)
>         put_amf_double(pb, audio_enc->codec_tag);
>     }
>
> +    if (data_enc) {
> +        put_amf_string(pb, "datastream");
> +        put_amf_double(pb, 0.0);
> +    }
> +

Is this a convention used by anything else other than this muxer/demuxer 
pair?

>     while ((tag = av_dict_get(s->metadata, "", tag, AV_DICT_IGNORE_SUFFIX))) {
>         put_amf_string(pb, tag->key);
>         avio_w8(pb, AMF_DATA_TYPE_STRING);
> @@ -396,7 +419,8 @@ static int flv_write_packet(AVFormatContext *s, AVPacket *pkt)
>     else
>         flags_size= 1;
>
> -    if (enc->codec_type == AVMEDIA_TYPE_VIDEO) {
> +    switch (enc->codec_type) {
> +    case AVMEDIA_TYPE_VIDEO:
>         avio_w8(pb, FLV_TAG_TYPE_VIDEO);
>
>         flags = enc->codec_tag;
> @@ -406,15 +430,22 @@ static int flv_write_packet(AVFormatContext *s, AVPacket *pkt)
>         }
>
>         flags |= pkt->flags & AV_PKT_FLAG_KEY ? FLV_FRAME_KEY : FLV_FRAME_INTER;
> -    } else {
> -        assert(enc->codec_type == AVMEDIA_TYPE_AUDIO);
> +    break;
> +    case AVMEDIA_TYPE_AUDIO:
>         flags = get_audio_flags(s, enc);
>
>         assert(size);
>
>         avio_w8(pb, FLV_TAG_TYPE_AUDIO);
> -    }
> +    break;
>
> +    case AVMEDIA_TYPE_DATA:
> +        avio_w8(pb, FLV_TAG_TYPE_META);
> +    break;
> +    default:
> +        return AVERROR(EINVAL);
> +    break;
> +    }
>     if (enc->codec_id == CODEC_ID_H264) {
>         /* check if extradata looks like MP4 */
>         if (enc->extradata_size > 0 && *(uint8_t*)enc->extradata != 1) {
> @@ -446,7 +477,31 @@ static int flv_write_packet(AVFormatContext *s, AVPacket *pkt)
>     avio_wb24(pb,ts);
>     avio_w8(pb,(ts >> 24) & 0x7F); // timestamps are 32bits _signed_
>     avio_wb24(pb,flv->reserved);
> +
> +    if (enc->codec_type == AVMEDIA_TYPE_DATA) {
> +        int data_size;
> +        int metadata_size_pos = avio_tell(pb);
> +        avio_w8(pb, AMF_DATA_TYPE_STRING);
> +        put_amf_string(pb, "onTextData");
> +        avio_w8(pb, AMF_DATA_TYPE_MIXEDARRAY);
> +        avio_wb32(pb, 2);
> +        put_amf_string(pb, "type");
> +        avio_w8(pb, AMF_DATA_TYPE_STRING);
> +        put_amf_string(pb, "Text");
> +        put_amf_string(pb, "text");
> +        avio_w8(pb, AMF_DATA_TYPE_STRING);
> +        put_amf_string(pb, pkt->data);
> +        put_amf_string(pb, "");
> +        avio_w8(pb, AMF_END_OF_OBJECT);
> +        /* write total size of tag */
> +        data_size = avio_tell(pb) - metadata_size_pos;
> +        avio_seek(pb, metadata_size_pos - 10, SEEK_SET);
> +        avio_wb24(pb, data_size);
> +        avio_seek(pb, data_size + 10 - 3, SEEK_CUR);
> +        avio_wb32(pb, data_size + 11);
> +    } else {
>     avio_w8(pb,flags);
> +
>     if (enc->codec_id == CODEC_ID_VP6)

Unrelated extra line :-)

>         avio_w8(pb,0);
>     if (enc->codec_id == CODEC_ID_VP6F)
> @@ -462,7 +517,7 @@ static int flv_write_packet(AVFormatContext *s, AVPacket *pkt)
>
>     avio_wb32(pb,size+flags_size+11); // previous tag size
>     flv->duration = FFMAX(flv->duration, pkt->pts + flv->delay + pkt->duration);
> -
> +    }
>     avio_flush(pb);
>
>     av_free(data);
> -- 
> 1.7.8.rc1

Other than the nitpicks and things to clarify, the actual code is ok for 
me.

// Martin


More information about the libav-devel mailing list