arduino-audio-tools
Loading...
Searching...
No Matches
Public Member Functions | Protected Member Functions | Static Protected Member Functions | Protected Attributes | Static Protected Attributes | List of all members

Video track configuration for MuxerMP4 - update before calling begin(). More...

#include <ContainerMP4.h>

Inheritance diagram for MuxerMP4:
Muxer VideoOutput

Public Member Functions

 MuxerMP4 ()
 
 MuxerMP4 (Print &out)
 
size_t addAudioFrame (const uint8_t *data, size_t len) override
 
size_t addI420Frame (const uint8_t *data, size_t len) override
 
size_t addJpegFrame (const uint8_t *data, size_t len) override
 
size_t addRGB565Frame (const uint8_t *data, size_t len) override
 
size_t addVideoFrame (const uint8_t *data, size_t len, bool isKeyFrame=true) override
 
size_t addYUV422Frame (const uint8_t *data, size_t len) override
 
uint32_t audioFrameCount ()
 Number of audio frames (fragments) written so far.
 
AudioInfoFormataudioInfo () override
 Provides read/write access to the audio track's AudioInfoFormat.
 
bool begin () override
 
void end () override
 Closes the encoder: no trailer is required for playback.
 
virtual void flush ()
 
float getAudioSamplesPerVideoFrame ()
 
MuxerVideoConfig getVideoInfo () override
 Provides the video track configuration.
 
const char * mime () override
 
 operator bool () override
 
void setAudioInfo (AudioInfoFormat info) override
 
void setAudioProfile (int aacProfile)
 
void setOutput (Print &out) override
 Defines the output: e.g. a local File or a network Client.
 
void setStreamType (StreamContentType type) override
 
void setVideoInfo (MuxerVideoConfig config) override
 Defines the video track configuration - call before begin()
 
StreamContentType streamType () override
 The track write() currently targets (see setStreamType())
 
uint32_t videoFrameCount ()
 Number of video frames (fragments) written so far.
 
size_t write (const uint8_t *data, size_t len) override
 

Protected Member Functions

size_t addRawFrame (const uint8_t *data, size_t len)
 
void checkRawFrame (VideoFormat expected, size_t len)
 
void checkVideoFormat (VideoFormat expected)
 
void setVideoConfigData (const uint8_t *spsPpsAnnexB, size_t len)
 
bool tryWriteMoov ()
 
void writeAudioSampleEntryHeader (MP4BoxWriter &b)
 
void writeAudioStbl (MP4BoxWriter &b)
 
void writeAvcC (MP4BoxWriter &b)
 
void writeDinf (MP4BoxWriter &b)
 
void writeEsds (MP4BoxWriter &b)
 
void writeEsdsMjpeg (MP4BoxWriter &b)
 
void writeFtypMoov ()
 
void writeHdlr (MP4BoxWriter &b, const char *handlerType, const char *name)
 
void writeMdhd (MP4BoxWriter &b, uint32_t timescale)
 
void writeMoofMdat (uint32_t trackId, const uint8_t *data, size_t len, uint32_t sampleDuration, bool isKeyFrame, uint32_t baseTime)
 
void writeMvex (MP4BoxWriter &b)
 
void writeMvhd (MP4BoxWriter &b)
 
void writeTkhd (MP4BoxWriter &b, uint32_t trackId, bool isVideo)
 
void writeTrak (MP4BoxWriter &b, uint32_t trackId, bool isVideo)
 
void writeVideoStbl (MP4BoxWriter &b)
 

Static Protected Member Functions

static int aacSampleRateIndex (uint32_t sampleRate)
 
template<typename F >
static void forEachAnnexBNal (const uint8_t *data, size_t len, F callback)
 

Protected Attributes

uint32_t audio_base_time = 0
 
uint32_t audio_frame_count = 0
 
AudioInfoFormat audio_info
 
int audio_profile = 2
 
uint32_t audio_sample_duration = 1024
 
uint32_t audio_seq = 0
 
uint32_t audio_timescale = 0
 
MP4BoxWriter box
 scratch buffer, reused for 'moov' and each 'moof'
 
uint32_t fragment_seq = 0
 
bool has_audio = false
 
bool is_open = false
 
bool moov_written = false
 
Vector< uint8_t > nal_tmp
 scratch buffer for Annex-B -> AVCC conversion
 
Printp_out = nullptr
 
Printp_print = nullptr
 
Vector< uint8_t > pps_data
 
Vector< uint8_t > sps_data
 
uint32_t video_base_time = 0
 
MuxerVideoConfig video_cfg
 
uint32_t video_frame_count = 0
 
uint32_t video_sample_duration = 0
 
uint32_t video_seq = 0
 
uint32_t video_timescale = 90000
 
StreamContentType write_stream_type = StreamContentType::Video
 

Static Protected Attributes

static const uint32_t kAudioTrackId = 2
 
static const uint32_t kVideoTrackId = 1
 

Detailed Description

Video track configuration for MuxerMP4 - update before calling begin().

Fragmented MP4 (fMP4) container encoder: muxes an already-encoded H.264 video stream (Annex-B access units, converted internally to AVCC for the 'mdat' samples) and an optional raw AAC audio stream into a fragmented MP4 stream written to a Print (a local File to record, or e.g. a network Client to publish a live stream to an HTTP/TCP client).

Unlike a classic single-'moov' MP4 (which needs the complete stsz/stco sample tables known upfront, and thus either a seekable output to place 'moov' before 'mdat', or buffering the entire recording in memory), fragmented MP4 writes 'ftyp'+'moov' once (with empty sample tables plus an 'mvex' box that tells the player to expect fragments) and then a self-contained 'moof'+'mdat' pair per frame - no seeking required, the same Print-only, forward-only design DemuxerMP4 already assumes on the read side and MuxerAVI uses for AVI.

Usage:

MuxerMP4 mux(client); // any Print: File, WiFiClient, ...
cfg.width = 640;
cfg.height = 480;
cfg.fps = 25;
cfg.format = VideoFormat::H264; // or MJPEG/YUV422/RGB565/I420
mux.setVideoInfo(cfg);
mux.setAudioInfo(AudioInfoFormat(44100, 1, 16)); // optional; AAC by
// default, or pass
// AudioFormat::PCM as
// the 4th argument
mux.begin();
// for each encoded H.264 access unit (Annex-B) - SPS/PPS are captured
// automatically, no separate setup call needed:
mux.addVideoFrame(h264_data, h264_len);
// for each raw (ADTS-less) AAC frame:
mux.addAudioFrame(aac_data, aac_len);
// Alternatively, drive both tracks through the single write() call
// (e.g. from generic StreamCopy-style code, mirroring MuxerAVI):
mux.setStreamType(StreamContentType::Video);
mux.write(h264_data, h264_len);
mux.setStreamType(StreamContentType::Audio);
mux.write(aac_data, aac_len); // audio: each write() is one full frame
Video track configuration for MuxerMP4 - update before calling begin().
Definition ContainerMP4.h:1163
@ Audio
Definition Video.h:19
@ Video
Definition Video.h:19
AudioInfo extended with a WAVEFORMATEX-style codec tag (the "wav code"): identifies the codec (PCM,...
Definition AudioFormat.h:389
Shared video track configuration for muxers (MuxerAVI, MuxerMP4) - call before begin().
Definition ContainerCommon.h:11
float fps
Definition ContainerCommon.h:14
uint16_t height
Definition ContainerCommon.h:13
uint16_t width
Definition ContainerCommon.h:12
VideoFormat format
Definition ContainerCommon.h:15
Note
v1 scope: one video track, either VideoFormat::H264 (default - matching the only video codec DemuxerMP4 reads back; write via addVideoFrame(), Annex-B access units) or one of the same raw/MJPEG formats MuxerAVI supports (DemuxerMP4 cannot read any of these back - muxer-only for now): VideoFormat::MJPEG (addJpegFrame(), one complete JPEG image per frame; 'mp4v'/'esds' sample entry, objectTypeIndication 0x6C), VideoFormat::YUV422 (addYUV422Frame(), packed YUY2/YUYV; 'yuvs' sample entry), VideoFormat::RGB565 (addRGB565Frame(), 16-bit 5-6-5; 'L565' sample entry), or VideoFormat::I420 (addI420Frame(), planar 4:2:0; 'I420' sample entry). All five formats were verified to produce a file ffprobe/ffmpeg parses and decodes cleanly (probe_score 100) - see the individual addXxxFrame() methods for exactly what was checked against real ffmpeg-generated reference data. Plus one optional audio track, either AAC (the default - raw, ADTS-less frames, matching the 'mdat' sample convention MP4 itself uses, not the ADTS-wrapped convention DemuxerMP4 hands to its own audio output; config limited to the common 2-byte AudioSpecificConfig case, LC profile, standard sample rates) or raw PCM (setAudioInfo() with format = AudioFormat::PCM - little-endian signed samples, 'sowt' sample entry, no encoder needed; only 16-bit has been validated). No mfra index / seeking metadata is written (not required for playback). SPS/PPS (needed for H264's 'avcC') are captured automatically from the first video frame(s) that carry them - typical H.264 encoders prepend them to (at least) the first access unit, so no separate setup call is needed; frames arriving before SPS/PPS have been seen are dropped (logged), as is any audio arriving before that same point, since 'moov'
  • which describes both tracks - cannot be written until then. MJPEG needs no such stream-derived config, so for it 'moov' is already written by the time begin() returns.
Author
Phil Schatzmann

Constructor & Destructor Documentation

◆ MuxerMP4() [1/2]

MuxerMP4 ( )
inline

◆ MuxerMP4() [2/2]

MuxerMP4 ( Print out)
inline

Member Function Documentation

◆ aacSampleRateIndex()

static int aacSampleRateIndex ( uint32_t  sampleRate)
inlinestaticprotected

◆ addAudioFrame()

size_t addAudioFrame ( const uint8_t *  data,
size_t  len 
)
inlineoverridevirtual

Writes one complete audio frame as a single 'moof'+'mdat' fragment. For AudioFormat::AAC (default): one raw (ADTS-less) AAC frame - the AAC config is described once via 'esds', unlike AVI/ADTS where it is repeated per frame. For AudioFormat::PCM: any number of interleaved PCM samples (e.g. one read buffer's worth) - the fragment's duration is derived from len, so callers may pass differently-sized chunks from call to call. Like addVideoFrame(), frames arriving before 'moov' can be written (i.e. before the video track's SPS/PPS have been seen) are dropped (logged).

Implements Muxer.

◆ addI420Frame()

size_t addI420Frame ( const uint8_t *  data,
size_t  len 
)
inlineoverridevirtual

Writes one planar 4:2:0 YUV (I420/IYUV: full-res Y, then quarter-res U, then quarter-res V) frame as a single 'moof'+'mdat' fragment. Expects exactly width*height*3/2 bytes - mismatches are logged, not rejected. No-op (returns 0) unless getVideoInfo().format is VideoFormat::I420.

Note
Muxer-only; see addYUV422Frame()'s note - its 'I420' sample entry round-trips correctly through ffprobe/ffmpeg despite ffmpeg's own rawvideo encoder having no direct pix_fmt+tag combination that produces one (a limitation of that one encoder path, not of the 'I420' tag itself - a hand-built file with the correct box structure decodes fine).

Implements Muxer.

◆ addJpegFrame()

size_t addJpegFrame ( const uint8_t *  data,
size_t  len 
)
inlineoverridevirtual

Writes one complete Motion-JPEG frame (a full, already-encoded JPEG image, e.g. as produced by an ESP32-CAM or other hardware JPEG encoder) as a single 'moof'+'mdat' fragment - no-op (returns 0) unless getVideoInfo().format is VideoFormat::MJPEG.

Implements Muxer.

◆ addRawFrame()

size_t addRawFrame ( const uint8_t *  data,
size_t  len 
)
inlineprotected

Writes one complete frame's bytes through as-is (no NAL-style framing needed - used by MJPEG, which is self-delimiting, and the raw pixel formats YUV422/RGB565/I420) as a single 'moof'+'mdat' fragment. Every such frame is independently decodable, so it is always marked as a sync sample. None of these formats need stream-derived config (unlike H264's SPS/PPS), so 'moov' is already written by the time begin() returns - tryWriteMoov() here is just a safety net.

◆ addRGB565Frame()

size_t addRGB565Frame ( const uint8_t *  data,
size_t  len 
)
inlineoverridevirtual

Writes one uncompressed RGB565 (16-bit, 5-6-5) frame as a single 'moof'+'mdat' fragment. Expects exactly width*height*2 bytes - mismatches are logged, not rejected. No-op (returns 0) unless getVideoInfo().format is VideoFormat::RGB565.

Note
Muxer-only; see addYUV422Frame()'s note - its 'L565' sample entry matches what ffmpeg itself writes/reads for rgb565le-in-MOV.

Implements Muxer.

◆ addVideoFrame()

size_t addVideoFrame ( const uint8_t *  data,
size_t  len,
bool  isKeyFrame = true 
)
inlineoverridevirtual

Writes one complete H.264 access unit (Annex-B: SPS/PPS NALs, if present, are captured for 'avcC' - see setVideoConfigData() - and then stripped, since they end up described there instead) as a single 'moof'+'mdat' fragment. Until SPS/PPS have been seen (from this or an earlier call), 'moov' cannot be written yet, so the frame is dropped (logged) rather than buffered. No-op (returns 0) unless getVideoInfo().format is VideoFormat::H264 - use addJpegFrame() for VideoFormat::MJPEG instead.

Parameters
isKeyFramemarks the sample as a sync sample (IDR) in the fragment's 'trun' flags - matters for players seeking into the stream; pass false for P/B (non-IDR) frames if known.

Implements Muxer.

◆ addYUV422Frame()

size_t addYUV422Frame ( const uint8_t *  data,
size_t  len 
)
inlineoverridevirtual

Writes one packed 4:2:2 YUV frame (YUY2/YUYV byte order) as a single 'moof'+'mdat' fragment. Expects exactly width*height*2 bytes - mismatches are logged, not rejected. No-op (returns 0) unless getVideoInfo().format is VideoFormat::YUV422.

Note
Muxer-only (DemuxerMP4 cannot read this back). Its 'yuvs' sample entry matches what ffmpeg itself writes for YUY2-in-MOV, and a muxed file round-trips cleanly through ffprobe/ffmpeg (correct pix_fmt, probe_score 100, decodes without error).

Implements Muxer.

◆ audioFrameCount()

uint32_t audioFrameCount ( )
inline

Number of audio frames (fragments) written so far.

◆ audioInfo()

AudioInfoFormat & audioInfo ( )
inlineoverridevirtual

Provides read/write access to the audio track's AudioInfoFormat.

Implements Muxer.

◆ begin()

bool begin ( )
inlineoverridevirtual

Prepares the encoder. 'ftyp'+'moov' (with an 'mvex' box signalling fragments will follow) is not written yet at this point - SPS/PPS (needed for 'avcC', part of 'moov') are only known once captured from the video stream itself, so it is written lazily, right before the first fragment that can actually be produced (see addVideoFrame()). Call after configuring video (and audio, if any) and before writing any frames.

Implements Muxer.

◆ checkRawFrame()

void checkRawFrame ( VideoFormat  expected,
size_t  len 
)
inlineprotected

Validates both the configured format and (for fixed-size raw formats) that len matches the width*height based expectation.

◆ checkVideoFormat()

void checkVideoFormat ( VideoFormat  expected)
inlineprotected

◆ end()

void end ( )
inlineoverridevirtual

Closes the encoder: no trailer is required for playback.

Implements Muxer.

◆ flush()

virtual void flush ( )
inlinevirtualinherited

Finalizes the frame most recently written via one or more write() calls - see class comment. Default no-op for implementations that display/decode synchronously in write() instead.

Reimplemented from VideoOutput.

◆ forEachAnnexBNal()

template<typename F >
static void forEachAnnexBNal ( const uint8_t *  data,
size_t  len,
callback 
)
inlinestaticprotected

Calls callback(nalDataPtr, nalDataLen) for every NAL unit found in an Annex-B buffer (data spans exclude the 00 00 01 / 00 00 00 01 start code itself).

◆ getAudioSamplesPerVideoFrame()

float getAudioSamplesPerVideoFrame ( )
inlineinherited

Average number of audio samples per video frame, derived from audioInfo().sample_rate and getVideoInfo().fps - the natural audio chunk size to write once per video frame if you want to keep both tracks advancing at roughly the same pace as you write them (not a hard requirement - see addAudioFrame()/addVideoFrame()). 0 if fps hasn't been set.

◆ getVideoInfo()

MuxerVideoConfig getVideoInfo ( )
inlineoverridevirtual

Provides the video track configuration.

Implements Muxer.

◆ mime()

const char * mime ( )
inlineoverridevirtual

The container's MIME type (e.g. "video/avi", "video/mp4") - useful for e.g. an HTTP Content-Type header when streaming the muxed output to a client.

Implements Muxer.

◆ operator bool()

operator bool ( )
inlineoverridevirtual

Implements Muxer.

◆ setAudioInfo()

void setAudioInfo ( AudioInfoFormat  info)
inlineoverridevirtual

Adds an (optional) interleaved audio track. 'info.format' selects the audio track's codec: AudioFormat::AAC (the default if left unset) writes an 'mp4a'/'esds' sample entry and expects raw (ADTS-less) AAC frames via addAudioFrame(); AudioFormat::PCM writes a 'sowt' (little-endian signed PCM) sample entry instead and expects raw interleaved PCM sample data - simplest option since it needs no encoder at all, but only 16-bit signed PCM has been validated. Call before begin().

Implements Muxer.

◆ setAudioProfile()

void setAudioProfile ( int  aacProfile)
inline

Defines the AAC profile (audio object type, default 2 = AAC LC) used to build the 'esds' AudioSpecificConfig - call before begin(). Not used for AudioFormat::PCM.

◆ setOutput()

void setOutput ( Print out)
inlineoverridevirtual

Defines the output: e.g. a local File or a network Client.

Implements Muxer.

◆ setStreamType()

void setStreamType ( StreamContentType  type)
inlineoverridevirtual

Selects whether write() feeds the video or the audio track, mirroring MuxerAVI's setStreamType(). Defaults to StreamContentType::Video; switch to StreamContentType::Audio (and back) around calls when using MuxerMP4 as a plain sink for both.

Implements Muxer.

◆ setVideoConfigData()

void setVideoConfigData ( const uint8_t *  spsPpsAnnexB,
size_t  len 
)
inlineprotected

Scans an Annex-B video frame for SPS/PPS NALs and captures them (for the 'avcC' box) if found - called automatically from addVideoFrame() for every frame, so whichever frame(s) happen to carry them (typically the first, and/or every keyframe) fill in the config. Does not clear previously-captured data when a frame contains neither.

◆ setVideoInfo()

void setVideoInfo ( MuxerVideoConfig  config)
inlineoverridevirtual

Defines the video track configuration - call before begin()

Implements Muxer.

◆ streamType()

StreamContentType streamType ( )
inlineoverridevirtual

The track write() currently targets (see setStreamType())

Implements Muxer.

◆ tryWriteMoov()

bool tryWriteMoov ( )
inlineprotected

Writes 'ftyp'+'moov', once SPS/PPS are available for H264 (MJPEG needs no equivalent stream-derived config, so it's ready immediately)

  • a no-op once already written. Returns true if 'moov' has been (now or previously) written, i.e. fragments may be written.

◆ videoFrameCount()

uint32_t videoFrameCount ( )
inline

Number of video frames (fragments) written so far.

◆ write()

size_t write ( const uint8_t *  data,
size_t  len 
)
inlineoverridevirtual

VideoOutput API / generic sink: writes one complete frame to whichever track streamType() currently selects (see setStreamType()). Each call is dispatched immediately - the caller must hand over the complete frame in a single call (video: to the addXxxFrame() matching getVideoInfo().format; audio: to addAudioFrame()). flush() is a no-op; there is no accumulation across write() calls.

Implements VideoOutput.

◆ writeAudioSampleEntryHeader()

void writeAudioSampleEntryHeader ( MP4BoxWriter b)
inlineprotected

Writes the common 28-byte AudioSampleEntry fixed header shared by 'mp4a' (AAC) and 'sowt' (PCM) - the same layout DemuxerMP4 itself reads (see its setupAudioInfo()): reserved+data_reference_index, reserved, channelcount, samplesize, pre_defined+reserved, samplerate.

◆ writeAudioStbl()

void writeAudioStbl ( MP4BoxWriter b)
inlineprotected

◆ writeAvcC()

void writeAvcC ( MP4BoxWriter b)
inlineprotected

◆ writeDinf()

void writeDinf ( MP4BoxWriter b)
inlineprotected

◆ writeEsds()

void writeEsds ( MP4BoxWriter b)
inlineprotected

◆ writeEsdsMjpeg()

void writeEsdsMjpeg ( MP4BoxWriter b)
inlineprotected

'esds' for the 'mp4v' MJPEG sample entry: objectTypeIndication 0x6C (ISO/IEC 10918-1, i.e. JPEG) with no DecoderSpecificInfo, since each JPEG sample is fully self-contained (its own quantization/Huffman tables) - unlike AAC, there is no shared out-of-band config. Layout verified against what ffmpeg itself writes for MJPEG-in-MP4.

◆ writeFtypMoov()

void writeFtypMoov ( )
inlineprotected

◆ writeHdlr()

void writeHdlr ( MP4BoxWriter b,
const char *  handlerType,
const char *  name 
)
inlineprotected

◆ writeMdhd()

void writeMdhd ( MP4BoxWriter b,
uint32_t  timescale 
)
inlineprotected

◆ writeMoofMdat()

void writeMoofMdat ( uint32_t  trackId,
const uint8_t *  data,
size_t  len,
uint32_t  sampleDuration,
bool  isKeyFrame,
uint32_t  baseTime 
)
inlineprotected

Writes one 'moof' (mfhd + traf{tfhd,tfdt,trun}) + 'mdat' fragment for a single sample. data_offset in 'trun' is relative to the start of 'moof' (the "default-base-is-moof" convention) - computed by first assembling 'moof' in the scratch buffer (its size is independent of the sample bytes themselves) and backpatching the offset field.

◆ writeMvex()

void writeMvex ( MP4BoxWriter b)
inlineprotected

◆ writeMvhd()

void writeMvhd ( MP4BoxWriter b)
inlineprotected

◆ writeTkhd()

void writeTkhd ( MP4BoxWriter b,
uint32_t  trackId,
bool  isVideo 
)
inlineprotected

◆ writeTrak()

void writeTrak ( MP4BoxWriter b,
uint32_t  trackId,
bool  isVideo 
)
inlineprotected

◆ writeVideoStbl()

void writeVideoStbl ( MP4BoxWriter b)
inlineprotected

Member Data Documentation

◆ audio_base_time

uint32_t audio_base_time = 0
protected

◆ audio_frame_count

uint32_t audio_frame_count = 0
protected

◆ audio_info

AudioInfoFormat audio_info
protected

◆ audio_profile

int audio_profile = 2
protected

◆ audio_sample_duration

uint32_t audio_sample_duration = 1024
protected

◆ audio_seq

uint32_t audio_seq = 0
protected

◆ audio_timescale

uint32_t audio_timescale = 0
protected

◆ box

MP4BoxWriter box
protected

scratch buffer, reused for 'moov' and each 'moof'

◆ fragment_seq

uint32_t fragment_seq = 0
protected

◆ has_audio

bool has_audio = false
protected

◆ is_open

bool is_open = false
protected

◆ kAudioTrackId

const uint32_t kAudioTrackId = 2
staticprotected

◆ kVideoTrackId

const uint32_t kVideoTrackId = 1
staticprotected

◆ moov_written

bool moov_written = false
protected

True once 'ftyp'+'moov' has actually been written (deferred until SPS/PPS are known - see tryWriteMoov()).

◆ nal_tmp

Vector<uint8_t> nal_tmp
protected

scratch buffer for Annex-B -> AVCC conversion

◆ p_out

Print* p_out = nullptr
protected

◆ p_print

Print* p_print = nullptr
protectedinherited

◆ pps_data

Vector<uint8_t> pps_data
protected

◆ sps_data

Vector<uint8_t> sps_data
protected

◆ video_base_time

uint32_t video_base_time = 0
protected

◆ video_cfg

MuxerVideoConfig video_cfg
protected

◆ video_frame_count

uint32_t video_frame_count = 0
protected

◆ video_sample_duration

uint32_t video_sample_duration = 0
protected

◆ video_seq

uint32_t video_seq = 0
protected

◆ video_timescale

uint32_t video_timescale = 90000
protected

◆ write_stream_type

StreamContentType write_stream_type = StreamContentType::Video
protected

The documentation for this class was generated from the following file: