arduino-audio-tools
Loading...
Searching...
No Matches
Classes | Public Types | Public Member Functions | Public Attributes | Protected Member Functions | Static Protected Member Functions | Protected Attributes | List of all members

DemuxerMP4 extracts both the audio and (H.264) video track from a general, interleaved MP4/ISO-BMFF stream - unlike M4AAudioDemuxer/ M4AAudioFileDemuxer, which only handle a single (audio) track. More...

#include <ContainerMP4.h>

Inheritance diagram for DemuxerMP4:
Demuxer ContainerDecoder AudioDecoder AudioWriter AudioInfoSource AudioInfoSupport

Classes

struct  ScheduleEntry
 
struct  StscEntry
 sample-to-chunk table entry (stsc) More...
 
struct  SttsEntry
 
struct  Track
 One track (audio or video) as found in 'moov'. More...
 

Public Types

using Codec = M4ACommonDemuxer::Codec
 

Public Member Functions

 DemuxerMP4 ()
 
 DemuxerMP4 (Print &video_out, Print &audio_out)
 
 DemuxerMP4 (VideoOutput &video_out, Print &audio_out)
 
 ~DemuxerMP4 ()
 
virtual void addNotifyAudioChange (AudioInfoSupport &bi)
 Adds target to be notified about audio changes.
 
Vector< uint8_t > & audioALACMagicCookie ()
 
AudioInfo audioInfo () override
 provides the actual input AudioInfo
 
virtual AudioInfo audioInfoOut ()
 
bool begin () override
 
virtual bool begin (AudioInfo info) override
 
virtual void clearNotifyAudioChange ()
 Deletes all change notify subscriptions.
 
void end () override
 
AudioInfoFormat getAudioInfo () override
 
PrintgetOutput ()
 
VideoInfo getVideoInfo () override
 
bool isNotifyActive ()
 Checks if the automatic AudioInfo update is active.
 
const char * mime () override
 The container's MIME type (e.g. "video/avi", "video/mp4").
 
 operator bool () override
 
virtual bool removeNotifyAudioChange (AudioInfoSupport &bi)
 Removes a target in order not to be notified about audio changes.
 
void setAudioInfo (AudioInfo from) override
 for most decoders this is not needed
 
virtual bool setCodecConfig (const uint8_t *data, size_t len)
 Some decoders need e.g. a magic cookie to provide the relevant info for decoding.
 
void setNotifyActive (bool flag)
 Deactivate/Reactivate automatic AudioInfo updates: (default is active)
 
virtual void setOutput (AudioOutput &out_stream)
 Defines where the decoded result is written to.
 
virtual void setOutput (AudioStream &out_stream)
 Defines where the decoded result is written to.
 
void setOutput (Print &out) override
 
void setOutputAudio (Print &out) override
 
void setOutputVideo (Print &out) override
 
void setOutputVideo (VideoOutput &out)
 
size_t write (const uint8_t *data, size_t len) override
 

Public Attributes

int id
 custom id to be used by application
 

Protected Member Functions

void advanceScheduleIfNeeded ()
 Advances to the next schedule entry once the current one is exhausted.
 
bool beginBoxAccum (const MP4Parser::Box &box)
 
void buildSchedule ()
 
void dispatchAudio (Track &t, const uint8_t *data, size_t size, bool isFirst)
 
void dispatchSample (Track &t, const uint8_t *data, size_t size, bool isFirst)
 
void dispatchVideo (Track &t, const uint8_t *data, size_t size, bool isFirst)
 
void endBoxAccum ()
 Call once a box_accum-using handler has fully processed a complete box.
 
void feed (const uint8_t *data, size_t len)
 
void flushVideo ()
 
void freeTracks ()
 
void notifyAudioChange (AudioInfo info)
 
void onAlac (const MP4Parser::Box &box)
 
void onAvc1 (const MP4Parser::Box &box)
 
void onAvcC (const MP4Parser::Box &box)
 
void onEsds (const MP4Parser::Box &box)
 
void onHdlr (MP4Parser::Box &box)
 
void onHevc (const MP4Parser::Box &box)
 
void onMdat (MP4Parser::Box &box)
 
void onMdhd (MP4Parser::Box &box)
 
void onMp4a (const MP4Parser::Box &box)
 
void onStco (MP4Parser::Box &box, bool is64)
 
void onStsc (MP4Parser::Box &box)
 
void onStsd (MP4Parser::Box &box)
 
void onStsz (MP4Parser::Box &box)
 
void onStts (MP4Parser::Box &box)
 
void onTrak ()
 
uint32_t samplesInChunk (Track &t, uint32_t chunkIndex0)
 Number of samples in the given (0-based) chunk index, per 'stsc'.
 
void sendWavHeader ()
 
void setSendWavHeader (bool flag) override
 
void setupAudioInfo (AudioFormat format, const uint8_t *entryData, size_t entrySize)
 
void setupParser ()
 
void writeBlocking (Print *out, uint8_t *data, size_t len)
 
void writeVideo (uint8_t *data, size_t size)
 

Static Protected Member Functions

static uint16_t readU16 (const uint8_t *p)
 
static uint32_t readU32 (const uint8_t *p)
 
static uint64_t readU64 (const uint8_t *p)
 
static void writeAdtsHeader (uint8_t *adts, int aacProfile, int sampleRateIdx, int channelCfg, int frameLen)
 

Protected Attributes

AudioFormat audio_format = AudioFormat::UNKNOWN
 
SingleBuffer< uint8_t > box_accum {0}
 
bool box_accum_active = false
 
bool current_sample_is_first_for_track = false
 
uint32_t current_sample_size = 0
 
Trackcurrent_sample_track = nullptr
 
Trackcurrent_track
 track currently being parsed (scoped by 'trak')
 
AudioInfo info
 
bool is_active = false
 
bool is_notify_active = true
 
bool is_wav_header_sent = false
 
bool mdat_seen = false
 
Vector< uint8_t > nal_tmp
 scratch buffer for Annex-B conversion
 
Vector< AudioInfoSupport * > notify_vector
 
Trackp_audio_track = nullptr
 first audio track found
 
Printp_output_audio = nullptr
 
Printp_print = nullptr
 
Printp_video_out = nullptr
 
VideoOutputp_video_out_video = nullptr
 
Trackp_video_track = nullptr
 first video track found
 
MP4Parser parser
 
uint32_t playback_start_ms = 0
 
bool playback_start_set = false
 
Vector< uint8_t > sample_buffer
 
size_t sample_buffer_filled = 0
 
uint32_t samples_left_in_entry = 0
 
Vector< ScheduleEntryschedule
 
size_t schedule_index = 0
 index into 'schedule'
 
bool send_wav_header = false
 
bool send_wav_header_explicit = false
 
bool stsz_header_pending = true
 
uint32_t total_bytes_received = 0
 
Vector< Track * > tracks
 

Detailed Description

DemuxerMP4 extracts both the audio and (H.264) video track from a general, interleaved MP4/ISO-BMFF stream - unlike M4AAudioDemuxer/ M4AAudioFileDemuxer, which only handle a single (audio) track.

This is a streaming (forward-only) demuxer: it does not need a seekable source, so it works directly over e.g. an HTTP download (URLStream) or an SD file read sequentially with StreamCopy - the same way DemuxerAVI is used. It relies on the file being "faststart" muxed (moov before mdat, e.g. ffmpeg ... -movflags +faststart) - the same requirement MP4Parser and the M4A demuxers already document.

How the interleaving is resolved without seeking: once 'moov' has been fully parsed, every track's stsz (sample sizes) + stsc (sample-to-chunk)

Note
v1 scope: one video track (H.264/avc1 only - HEVC is detected and skipped with a warning) and one audio track (AAC or ALAC). Video output is the raw H.264 Annex-B elementary stream (SPS/PPS from avcC converted to Annex-B and prepended before the first frame) via VideoOutput - there is no pixel decoder anywhere in this library, same scope boundary as DemuxerAVI. No stts/timing support yet - write() dispatches samples as soon as they are complete; pacing is the caller's responsibility.
Author
Phil Schatzmann

Member Typedef Documentation

◆ Codec

Constructor & Destructor Documentation

◆ DemuxerMP4() [1/3]

DemuxerMP4 ( )
inline

This class only demuxes - it does not decode audio itself. Point setOutputAudio() at an EncodedAudioStream (wrapping whatever AudioDecoder matches the track's codec - AAC/ALAC) if you need the audio track decoded; the raw payload (ADTS-wrapped for AAC, raw for ALAC) is written through as-is otherwise. For ALAC, the decoder also needs the magic cookie exposed via audioALACMagicCookie() - configure your decoder with it (e.g. AudioDecoder::setCodecConfig()) once it's available (after the audio track's 'stsd' has been parsed, i.e. once getAudioInfo().format != AudioFormat::UNKNOWN). Point setOutputVideo() at a Print (e.g. a VideoOutput) to receive the demuxed H.264 Annex-B video track; leave unset to ignore video.

◆ DemuxerMP4() [2/3]

DemuxerMP4 ( Print video_out,
Print audio_out 
)
inline

◆ DemuxerMP4() [3/3]

DemuxerMP4 ( VideoOutput video_out,
Print audio_out 
)
inline

◆ ~DemuxerMP4()

~DemuxerMP4 ( )
inline

Member Function Documentation

◆ addNotifyAudioChange()

virtual void addNotifyAudioChange ( AudioInfoSupport bi)
inlinevirtualinherited

◆ advanceScheduleIfNeeded()

void advanceScheduleIfNeeded ( )
inlineprotected

Advances to the next schedule entry once the current one is exhausted.

◆ audioALACMagicCookie()

Vector< uint8_t > & audioALACMagicCookie ( )
inline

ALAC magic cookie (the 'alac' box payload, without its own size/type prefix) needed to configure an external ALAC decoder - empty unless getAudioInfo().format == AudioFormat::ALAC.

◆ audioInfo()

AudioInfo audioInfo ( )
inlineoverridevirtualinherited

◆ audioInfoOut()

virtual AudioInfo audioInfoOut ( )
inlinevirtualinherited

◆ begin() [1/2]

bool begin ( )
inlineoverridevirtual

Reimplemented from AudioDecoder.

◆ begin() [2/2]

virtual bool begin ( AudioInfo  info)
inlineoverridevirtualinherited

Reimplemented from AudioWriter.

Reimplemented in DecoderBasic, and OggContainerDecoder.

◆ beginBoxAccum()

bool beginBoxAccum ( const MP4Parser::Box box)
inlineprotected

Call at the top of every box_accum-using handler. Returns true the first time it's called for a given box (i.e. right when accumulation starts), false on continuations - callers that need to (re)initialize per-box state (e.g. onStsz's stsz_header_pending) should key off that.

◆ buildSchedule()

void buildSchedule ( )
inlineprotected

Merge all tracks' (chunk_offset, samples_in_chunk) entries, sorted ascending by chunk_offset, into a single consumption order.

◆ clearNotifyAudioChange()

virtual void clearNotifyAudioChange ( )
inlinevirtualinherited

Deletes all change notify subscriptions.

Reimplemented in RTSPClient< TcpClient, UdpSocket >.

◆ dispatchAudio()

void dispatchAudio ( Track t,
const uint8_t *  data,
size_t  size,
bool  isFirst 
)
inlineprotected

◆ dispatchSample()

void dispatchSample ( Track t,
const uint8_t *  data,
size_t  size,
bool  isFirst 
)
inlineprotected

◆ dispatchVideo()

void dispatchVideo ( Track t,
const uint8_t *  data,
size_t  size,
bool  isFirst 
)
inlineprotected

◆ end()

void end ( )
inlineoverridevirtual

Reimplemented from AudioDecoder.

◆ endBoxAccum()

void endBoxAccum ( )
inlineprotected

Call once a box_accum-using handler has fully processed a complete box.

◆ feed()

void feed ( const uint8_t *  data,
size_t  len 
)
inlineprotected

Feeds raw mdat bytes; accumulates until the current sample is complete, dispatches it, and moves on - possibly switching tracks at chunk boundaries per the merge schedule.

◆ flushVideo()

void flushVideo ( )
inlineprotected

◆ freeTracks()

void freeTracks ( )
inlineprotected

◆ getAudioInfo()

AudioInfoFormat getAudioInfo ( )
inlineoverridevirtual

Common audio info (sample_rate/channels/bits_per_sample), extended with the parsed codec format tag - the same AudioInfoFormat type is also provided by DemuxerAVI. Named getAudioInfo() rather than audioInfo() because AudioDecoder already declares a virtual AudioInfo audioInfo() (returning the plain, unextended type by value) - by-value virtual returns can't be covariantly widened in C++, so reusing that name for a subclass-returning version isn't possible (it would be a hard "invalid covariant return type" error, not silent hiding).

Implements Demuxer.

◆ getOutput()

Print * getOutput ( )
inlineinherited

◆ getVideoInfo()

VideoInfo getVideoInfo ( )
inlineoverridevirtual

Common video info (width/height/format/frame_size/total_file_size), analogous to audioInfo() - the same VideoInfo type is also provided by DemuxerAVI. format is VideoFormat::H264 (the only video codec this class demuxes) once the video track's 'stsd' has been parsed, UNKNOWN before that or for an unsupported codec (e.g. HEVC). frame_size is always 0 (H264 frame size varies) - kept for symmetry with DemuxerAVI, whose raw pixel formats do have a fixed per-frame size. total_file_size has no declared-upfront equivalent in MP4 (no single header field states it, unlike AVI's RIFF header), so it reports the number of bytes received via write() so far instead - only equal to the true total once the whole stream has been fed in.

Implements Demuxer.

◆ isNotifyActive()

bool isNotifyActive ( )
inlineinherited

Checks if the automatic AudioInfo update is active.

◆ mime()

const char * mime ( )
inlineoverridevirtual

The container's MIME type (e.g. "video/avi", "video/mp4").

Implements Demuxer.

◆ notifyAudioChange()

void notifyAudioChange ( AudioInfo  info)
inlineprotectedinherited

◆ onAlac()

void onAlac ( const MP4Parser::Box box)
inlineprotected

◆ onAvc1()

void onAvc1 ( const MP4Parser::Box box)
inlineprotected

◆ onAvcC()

void onAvcC ( const MP4Parser::Box box)
inlineprotected

◆ onEsds()

void onEsds ( const MP4Parser::Box box)
inlineprotected

◆ onHdlr()

void onHdlr ( MP4Parser::Box box)
inlineprotected

◆ onHevc()

void onHevc ( const MP4Parser::Box box)
inlineprotected

◆ onMdat()

void onMdat ( MP4Parser::Box box)
inlineprotected

◆ onMdhd()

void onMdhd ( MP4Parser::Box box)
inlineprotected

'mdhd' (in 'mdia', scoped by the enclosing 'trak'): captures this track's own timescale (ticks/second), needed together with 'stts' to compute each sample's scheduled presentation time.

◆ onMp4a()

void onMp4a ( const MP4Parser::Box box)
inlineprotected

◆ onStco()

void onStco ( MP4Parser::Box box,
bool  is64 
)
inlineprotected

◆ onStsc()

void onStsc ( MP4Parser::Box box)
inlineprotected

◆ onStsd()

void onStsd ( MP4Parser::Box box)
inlineprotected

◆ onStsz()

void onStsz ( MP4Parser::Box box)
inlineprotected

◆ onStts()

void onStts ( MP4Parser::Box box)
inlineprotected

'stts' (time-to-sample): run-length list of {sample_count, sample_delta} - see Track::nextPtsTicks().

◆ onTrak()

void onTrak ( )
inlineprotected

◆ operator bool()

operator bool ( )
inlineoverridevirtual

Implements AudioWriter.

◆ readU16()

static uint16_t readU16 ( const uint8_t *  p)
inlinestaticprotected

◆ readU32()

static uint32_t readU32 ( const uint8_t *  p)
inlinestaticprotected

◆ readU64()

static uint64_t readU64 ( const uint8_t *  p)
inlinestaticprotected

◆ removeNotifyAudioChange()

virtual bool removeNotifyAudioChange ( AudioInfoSupport bi)
inlinevirtualinherited

Removes a target in order not to be notified about audio changes.

Reimplemented in RTSPClient< TcpClient, UdpSocket >.

◆ samplesInChunk()

uint32_t samplesInChunk ( Track t,
uint32_t  chunkIndex0 
)
inlineprotected

Number of samples in the given (0-based) chunk index, per 'stsc'.

◆ sendWavHeader()

void sendWavHeader ( )
inlineprotected

Synthesizes a valid WAV header (via the shared WAVHeader writer) from the parsed AudioInfo and sends it to p_output_audio, once, before any real audio payload - so a WAVDecoder-based output can bootstrap itself from it. Length is written as streamed/unknown, since the track's length is not known upfront while demuxing. Only ever relevant for a raw-PCM ('lpcm') audio track - not currently produced by this class (AAC/ALAC are the only codecs parsed), but kept symmetric with DemuxerAVI for when PCM tracks are added.

◆ setAudioInfo()

void setAudioInfo ( AudioInfo  from)
inlineoverridevirtualinherited

for most decoders this is not needed

Implements AudioWriter.

Reimplemented in DecoderALAC, CodecChain, DSFDecoder, DecoderL8, OpusAudioDecoder, OpusMultiStreamAudioDecoder, CodecNOP, G729Decoder, and G7xxDecoder.

◆ setCodecConfig()

virtual bool setCodecConfig ( const uint8_t *  data,
size_t  len 
)
inlinevirtualinherited

Some decoders need e.g. a magic cookie to provide the relevant info for decoding.

Reimplemented in DecoderALAC, and MultiDecoder.

◆ setNotifyActive()

void setNotifyActive ( bool  flag)
inlineinherited

Deactivate/Reactivate automatic AudioInfo updates: (default is active)

◆ setOutput() [1/3]

virtual void setOutput ( AudioOutput out_stream)
inlinevirtualinherited

Defines where the decoded result is written to.

Reimplemented in ADTSDecoder, CodecChain, MTSDecoder, MTSDecoderTSDemux, and MetaDataFilterDecoder.

◆ setOutput() [2/3]

virtual void setOutput ( AudioStream out_stream)
inlinevirtualinherited

Defines where the decoded result is written to.

Reimplemented in ADTSDecoder, CodecChain, MTSDecoder, MTSDecoderTSDemux, and MetaDataFilterDecoder.

◆ setOutput() [3/3]

void setOutput ( Print out)
inlineoverridevirtual

Satisfies the AudioWriter/AudioDecoder interface - needed so EncodedAudioOutput/EncodedAudioStream's polymorphic AudioDecoder* wiring (which calls setOutput() through that base class pointer) still reaches the audio output correctly; delegates to setOutputAudio().

Reimplemented from AudioDecoder.

◆ setOutputAudio()

void setOutputAudio ( Print out)
inlineoverridevirtual

Defines the audio output stream - e.g. an EncodedAudioStream wrapping an AudioDecoder that matches the audio track's codec, or any other Print if you want the raw payload as-is.

Implements Demuxer.

◆ setOutputVideo() [1/2]

void setOutputVideo ( Print out)
inlineoverridevirtual

Defines the video output - e.g. a VideoOutput implementation, or any other Print if you want the raw payload as-is.

Implements Demuxer.

◆ setOutputVideo() [2/2]

void setOutputVideo ( VideoOutput out)
inline

◆ setSendWavHeader()

void setSendWavHeader ( bool  flag)
inlineoverrideprotectedvirtual

Overrides the automatic decision of whether a synthetic WAV header is sent to the audio output before any audio payload. By default this is decided automatically from the parsed codec: on whenever isWavFormat() of the parsed format is true (raw PCM), off otherwise (AAC/ALAC, which have their own decoder and don't expect a WAV header). Call this to force it either way instead.

Implements Demuxer.

◆ setupAudioInfo()

void setupAudioInfo ( AudioFormat  format,
const uint8_t *  entryData,
size_t  entrySize 
)
inlineprotected

Populates the shared AudioInfo (channels/sample_rate/format/mime) from an AudioSampleEntry's fixed 28-byte header (ISO/IEC 14496-12 12.2.3): channelcount at rel-offset 16-17, samplerate (16.16 fixed point, upper 16 bits = Hz) at rel-offset 24-27 - the same layout for any audio sample entry (mp4a, alac, ...), so this is codec-agnostic.

◆ setupParser()

void setupParser ( )
inlineprotected

◆ write()

size_t write ( const uint8_t *  data,
size_t  len 
)
inlineoverridevirtual

Partial-write contract (like AVIDecoder::write()): MP4Parser's own internal buffer has finite capacity, so a single call may accept fewer bytes than requested - the caller is expected to call write() again with the remainder, exactly like the underlying Print contract.

Implements AudioWriter.

◆ writeAdtsHeader()

static void writeAdtsHeader ( uint8_t *  adts,
int  aacProfile,
int  sampleRateIdx,
int  channelCfg,
int  frameLen 
)
inlinestaticprotected

◆ writeBlocking()

void writeBlocking ( Print out,
uint8_t *  data,
size_t  len 
)
inlineprotectedinherited

◆ writeVideo()

void writeVideo ( uint8_t *  data,
size_t  size 
)
inlineprotected

Member Data Documentation

◆ audio_format

AudioFormat audio_format = AudioFormat::UNKNOWN
protected

Codec format tag of the (first) audio track - AudioFormat::UNKNOWN until its 'stsd' has been parsed. Kept separately from AudioInfo (see AudioInfoFormat) since AudioInfo itself has no format field.

◆ box_accum

SingleBuffer<uint8_t> box_accum {0}
protected

◆ box_accum_active

bool box_accum_active = false
protected

◆ current_sample_is_first_for_track

bool current_sample_is_first_for_track = false
protected

◆ current_sample_size

uint32_t current_sample_size = 0
protected

◆ current_sample_track

Track* current_sample_track = nullptr
protected

◆ current_track

Track* current_track
protected
Initial value:
=
nullptr

track currently being parsed (scoped by 'trak')

◆ id

int id
inherited

custom id to be used by application

◆ info

AudioInfo info
protectedinherited

◆ is_active

bool is_active = false
protected

◆ is_notify_active

bool is_notify_active = true
protectedinherited

◆ is_wav_header_sent

bool is_wav_header_sent = false
protected

◆ mdat_seen

bool mdat_seen = false
protected

◆ nal_tmp

Vector<uint8_t> nal_tmp
protected

scratch buffer for Annex-B conversion

◆ notify_vector

Vector<AudioInfoSupport*> notify_vector
protectedinherited

◆ p_audio_track

Track* p_audio_track = nullptr
protected

first audio track found

◆ p_output_audio

Print* p_output_audio = nullptr
protected

◆ p_print

Print* p_print = nullptr
protectedinherited

◆ p_video_out

Print* p_video_out = nullptr
protected

◆ p_video_out_video

VideoOutput* p_video_out_video = nullptr
protected

◆ p_video_track

Track* p_video_track = nullptr
protected

first video track found

◆ parser

MP4Parser parser
protected

◆ playback_start_ms

uint32_t playback_start_ms = 0
protected

◆ playback_start_set

bool playback_start_set = false
protected

◆ sample_buffer

Vector<uint8_t> sample_buffer
protected

◆ sample_buffer_filled

size_t sample_buffer_filled = 0
protected

◆ samples_left_in_entry

uint32_t samples_left_in_entry = 0
protected

◆ schedule

Vector<ScheduleEntry> schedule
protected

◆ schedule_index

size_t schedule_index = 0
protected

index into 'schedule'

◆ send_wav_header

bool send_wav_header = false
protected

◆ send_wav_header_explicit

bool send_wav_header_explicit = false
protected

◆ stsz_header_pending

bool stsz_header_pending = true
protected

◆ total_bytes_received

uint32_t total_bytes_received = 0
protected

Bytes received via write() so far this stream - MP4 has no declared total-file-size field to report instead (see getVideoInfo()).

◆ tracks

Vector<Track*> tracks
protected

The documentation for this class was generated from the following file: