c++ – Upsample and encode audio stream

Basically after transcoding pcm_alaw 8khz to mp3, I can hear only some brief or even swift sound in first 1-2 seconds, unrecognizable sound. So something is wrong with pts/dts, packed to planar convertion, or upsampling.

My application does transcoding rtsp camera stream to file. Video and audio. Video works fine and audio remuxing as well. Now I have pcm_alaw 8khz audio stream and want to transcode it to mp4 file along with video.

Code is quite cumbersome to construct reproducible part, so firstly I want to know if my logic is right. Here is my draft process (assume all error are checked and handled):

create encoder:

    codec_ = avcodec_find_encoder(AV_CODEC_ID_MP3);

    enc_ctx_ = avcodec_alloc_context3(codec_);

    enc_ctx_->bit_rate = 64000;
    enc_ctx_->codec_type = AVMEDIA_TYPE_AUDIO;

    enc_ctx_->sample_fmt   = codec_->sample_fmts ? codec_->sample_fmts[0] : AV_SAMPLE_FMT_S32P;

    // functions from here https://www.ffmpeg.org/doxygen/4.1/encode_audio_8c-example.html
    enc_ctx_->sample_rate    = select_sample_rate(codec_);
    enc_ctx_->channel_layout = select_channel_layout(codec_);
    enc_ctx_->channels       = av_get_channel_layout_nb_channels(enc_ctx_->channel_layout);
    enc_ctx_->time_base = (AVRational){1, enc_ctx_->sample_rate};
    enc_ctx_->strict_std_compliance = FF_COMPLIANCE_EXPERIMENTAL;

    if (is_global_header) {
        enc_ctx_->flags |= AV_CODEC_FLAG_GLOBAL_HEADER;
    }

    avcodec_open2(enc_ctx_, codec_, nullptr);

create resampler (in_frame):

    audio_fifo_ = av_audio_fifo_alloc(enc_ctx_->sample_fmt, enc_ctx_->channels, 1));
       
    in_ch_layout_ = in_frame->channel_layout;
    in_sample_fmt = in_frame->format;
    in_sample_rate_ = in_frame->sample_rate;

    swr_ctx_ = swr_alloc_set_opts(NULL,                       // we're allocating a new context
                             enc_ctx_->channel_layout,        // out_ch_layout
                             enc_ctx_->sample_fmt,            // out_sample_fmt
                             enc_ctx_->sample_rate,           // out_sample_rate
                             in_frame->channel_layout,        // in_ch_layout
                             (AVSampleFormat)in_frame->format, // in_sample_fmt
                             in_frame->sample_rate,            // in_sample_rate
                             0,                                // log_offset
                             NULL);                            // log_ctx
                             
    swr_init(swr_ctx_);

resample(in_frame, start_pts, start_dts):

    auto resampled_frame = av_frame_alloc();

    auto dst_nb_samples = av_rescale_rnd(swr_get_delay(swr_ctx_, in_frame->sample_rate) +
                                    in_frame->nb_samples, enc_ctx_->sample_rate, in_frame->sample_rate, AV_ROUND_UP);

    // resampled_frame->nb_samples     = dst_nb_samples;
    resampled_frame->format         = enc_ctx_->sample_fmt;
    resampled_frame->channel_layout = enc_ctx_->channel_layout;
    // resampled_frame->channels       = enc_ctx_->channels;
    resampled_frame->sample_rate    = enc_ctx_->sample_rate;

    error = swr_convert_frame(swr_ctx_, resampled_frame, in_frame);

    /* Make the FIFO as large as it needs to be to hold both,
     * the old and the new samples. */
    if (av_audio_fifo_size(audio_fifo_) < dst_nb_samples) {
        av_audio_fifo_realloc(audio_fifo_, dst_nb_samples);
    }

    /* Store the new samples in the FIFO buffer. */
    auto nb_samples = av_audio_fifo_write(audio_fifo_,
                                          reinterpret_cast<void **>(resampled_frame->extended_data),
                                          resampled_frame->nb_samples);


    int delay = 0;
    // trying to split resampled frame to desired chunks
    while (av_audio_fifo_size(audio_fifo_) > 0) {
        const int frame_size = FFMIN(av_audio_fifo_size(audio_fifo_), enc_ctx_->frame_size);

        auto out_frame = av_frame_alloc();


        out_frame->nb_samples       = frame_size;
        out_frame->format           = enc_ctx_->sample_fmt;
        out_frame->channel_layout   = enc_ctx_->channel_layout;
        out_frame->channels         = enc_ctx_->channels;
        out_frame->sample_rate      = enc_ctx_->sample_rate;

        av_frame_get_buffer(out_frame, 0);
        
        av_audio_fifo_read(audio_fifo_, (void **)out_frame->data, frame_size) < frame_size);

    // ***** tried both cases
        out_frame->pts = in_frame->pts + delay;
        out_frame->pkt_dts = in_frame->pkt_dts + delay;
        // swr_next_pts(swr_ctx_, in_frame->pts) + delay;
        // swr_next_pts(swr_ctx_, in_frame->pkt_dts) + delay;

        result.push_back(out_frame);

        delay += frame_size;
    }

    return result;

encoding and muxing (in_frame):

    bool DoesNeedResample(const AVFrame * in_frame) {
        assert(("DoesNeedResample: in_frame is empty", in_frame));
        assert(("DoesNeedResample: encoder is not started", is_init_));

        if (in_frame->sample_rate != enc_ctx_->sample_rate ||
        in_frame->channel_layout != enc_ctx_->channel_layout ||
        in_frame->channels != enc_ctx_->channels ||
        in_frame->format != enc_ctx_->sample_fmt) {
        return true;
        }

        return false;
    }

    av_frame_make_writable(in_frame);


    streamserver::AVFrames encoding_frames;
    if (DoesNeedResample(in_frame)) {
        encoding_frames = Resample(in_frame, 
        av_rescale_q(in_frame->pts, in_audio_stream_timebase_, out_audio_stream_->time_base),
        av_rescale_q(in_frame->pkt_dts, in_audio_stream_timebase_, out_audio_stream_->time_base));
    } else {
        encoding_frames.push_back(av_frame_clone(in_frame));
    }


    for (auto frame : encoding_frames) {
        frame->pts -= shift_pts_audio_;
        frame->pkt_dts -= shift_pts_audio_;

        if ((err = avcodec_send_frame(encoder_ctx, frame)) < 0) {
            AVFrameFree(&frame);
        }

        while (err >= 0) {
            pkt_->data = NULL;
            pkt_->size = 0;
            av_init_packet(pkt_);

            err = avcodec_receive_packet(encoder_ctx, pkt_);
            if (err == AVERROR(EAGAIN) || err == AVERROR_EOF) {
                break;
            } else if (err < 0) {
                break;
            }

            pkt_->stream_index = out_audio_stream_->index;

            av_interleaved_write_frame(ofmt_ctx_, pkt_);
        }

        av_packet_unref(pkt_);
    }

In https://www.ffmpeg.org/doxygen/4.1/transcode_aac_8c-example.html there are lines:

        /*
        * Perform a sanity check so that the number of converted samples is
        * not greater than the number of samples to be converted.
        * If the sample rates differ, this case has to be handled differently
        */
        av_assert0(output_codec_context->sample_rate == input_codec_context->sample_rate);

How to handle such cases? I tried to split resampled frames via fifo in example above!

Leave a Comment