H264Decoder.h

Go to the documentation of this file.

#pragma once
 
/**
 * @file H264Decoder.h
 * @brief ESP32-S3 H.264 Decoder with Stream Input Support
 * @version 1.0
 * @date 2024-12-19
 * @author Generated by GitHub Copilot 
 * 
 * Provides H264Decoder class for decoding H.264 video streams on ESP32-S3.
 * Supports various input sources via Arduino Stream interface and multiple 
 * output formats (I420, RGB565).
 * 
 * This is a lightweight, header-only helper to decode H.264 streams to raw frames
 * @author esp_h264 library
 */
 
#include <Arduino.h>
#include <esp_log.h>
#include <functional>
#include <vector>
#include <algorithm>
 
#include "H264Config.h"
#include "h264/esp_h264_arduino.h"
#include "h264/esp_h264_dec.h"
#include "h264/esp_h264_dec_sw.h"
#include "h264/esp_h264_dec_param.h"
#include "h264/esp_h264_types.h"
 
namespace esp_h264 {
 
/**
 * @class H264Decoder
 * @brief Template class for decoding H.264 streams to raw video frames
 * 
 * H264Decoder is a lightweight, header-only class that wraps H.264 decoder initialization,
 * stream processing, and output format conversions (I420 → RGB565/other formats).
 * 
 * The class is templated on an allocator type to allow customization of memory
 * allocation strategy (e.g., PSRAM vs regular heap).
 * 
 * @tparam Alloc Memory allocator type for internal buffers (defaults to PSRAMAllocatorH264)
 * 
 * @note This class intentionally keeps implementation in the header for easy 
 *       integration into Arduino sketches without separate .cpp files
 * @note The decoder supports constrained baseline H.264 profile
 * @note Requires ESP32 with sufficient memory for frame buffers
 * 
 * Example usage:
 * @code
 * #include "H264Decoder.h"
 * #include <WiFiUdp.h>
 * 
 * WiFiUDP udp;
 * H264DecoderPSRAM decoder(&udp);  // Uses PSRAM allocation (simpler than H264Decoder<>)
 * 
 * void setup() {
 *   auto cfg = decoder.defaultConfig();
 *   cfg.output_format = ESP_H264_RAW_FMT_RGB565_LE;
 *   cfg.frame_callback = [](const uint8_t* frame, uint32_t w, uint32_t h, esp_h264_raw_format_t fmt) {
 *     // Process decoded frame (e.g., display on TFT)
 *     tft.drawFrame(frame, w, h);
 *   };
 *   
 *   udp.begin(5000);
 *   if (!decoder.begin(cfg)) {
 *     Serial.println("Failed to start decoder");
 *   }
 * }
 * 
 * void loop() {
 *   decoder.processStream();  // Reads from UDP and decodes
 * }
 * @endcode
 */
template <typename Alloc = H264_DEFAULT_ALLOCATOR>
class H264Decoder {
 public:
  /** @brief Logging tag for ESP32 log system */
  static constexpr const char* TAG = "H264Decoder";
 
  /**
   * @struct Config
   * @brief Configuration structure for H.264 decoder and output settings
   * 
   * Contains all parameters needed to configure the H.264 decoder,
   * output format conversion, and buffer management.
   */
  struct Config {
    /** @brief Output pixel format after decoding (I420, RGB565, etc.) */
    esp_h264_raw_format_t output_format = ESP_H264_RAW_FMT_I420;
 
    /** @brief Input buffer size for H.264 stream data (bytes) */
    size_t input_buffer_size = 400 * 1024;
 
    /** @brief Output buffer size for decoded frame data (bytes) */
    size_t output_buffer_size = 640 * 480 * 3 / 2;  // Default for VGA I420
 
    /** @brief Maximum bytes to read per processStream() call (0 = read all available) */
    size_t max_read_bytes = 1500;  // Typical UDP packet size
 
    /** @brief Optional frame callback for real-time processing */
    std::function<void(const uint8_t* frame_data, uint32_t width, uint32_t height,
                      esp_h264_raw_format_t format)> frame_callback = nullptr;
  };
 
  /**
   * @brief Create a default configuration with reasonable defaults
   * 
   * Returns a Config structure populated with default values suitable for
   * typical H.264 decoding scenarios. Output format defaults to I420.
   * 
   * @return Config Default configuration structure
   * 
   * @note Input and output buffer sizes are set for VGA resolution
   * @note Frame callback is nullptr - set manually for real-time processing
   */
  Config defaultConfig() {
    ESP_LOGD(TAG, "defaultConfig");
    Config cfg;
    cfg.output_format = ESP_H264_RAW_FMT_I420;
    cfg.input_buffer_size = 400 * 1024;
    cfg.output_buffer_size = 640 * 480 * 3 / 2;
    cfg.max_read_bytes = 1500;
    cfg.frame_callback = nullptr;
    return cfg;
  }
 
  /**
   * @brief Constructor with input stream
   * 
   * Creates a H264Decoder that reads H.264 data from the specified stream.
   * The stream can be UDP, TCP, Serial, or any other Stream implementation.
   * 
   * @param input_stream Pointer to Stream object for H.264 input data
   * 
   * @note The stream must remain valid for the lifetime of the decoder
   * @note Call begin() to initialize decoder resources
   */
  explicit H264Decoder(Stream& input_stream) : input_stream_(&input_stream) {
    ESP_LOGD(TAG, "H264Decoder(Stream&)");
  }
 
  /**
   * @brief Default constructor (no input stream)
   * 
   * Creates a H264Decoder without an input stream. Use decode() method
   * to process H.264 data directly from buffers.
   * 
   * @note Call begin() to initialize decoder resources
   * @note Use setInputStream() to assign a stream later if needed
   */
  H264Decoder() : input_stream_(nullptr) { ESP_LOGD(TAG, "H264Decoder"); }
 
  /**
   * @brief Destructor
   * 
   * Automatically releases decoder and buffer resources by calling end().
   * 
   * @note Safe to destroy without calling end() explicitly
   */
  ~H264Decoder() {
    ESP_LOGD(TAG, "~H264Decoder");
    end();
  }
 
  /**
   * @brief Set input stream for H.264 data
   * 
   * Assigns or changes the input stream used by processStream() method.
   * 
   * @param input_stream Pointer to Stream object for H.264 input data
   * 
   * @note The stream must remain valid for the lifetime of the decoder
   * @note Can be called at any time to change input source
   */
  void setInputStream(Stream* input_stream) {
    ESP_LOGD(TAG, "setInputStream");
    input_stream_ = input_stream;
  }
 
  /**
   * @brief Initialize H.264 decoder and allocate buffers
   * 
   * Configures and initializes the H.264 software decoder with specified
   * output format and allocates internal buffers using the configured allocator.
   * 
   * @param cfg Configuration structure (passed by value to allow temporary objects)
   * @return true if initialization succeeded, false on any failure
   * 
   * @note Decoder is configured for constrained baseline H.264 profile
   * @note Buffers are allocated using the template allocator (Alloc)
   * @note On failure, any partially initialized resources are cleaned up
   */
  bool begin(Config cfg) {
    ESP_LOGD(TAG, "begin");
    cfg_ = cfg;
    if (!initDecoder()) return false;
    if (!initBuffers()) return false;
    return true;
  }
 
  /**
   * @brief Process H.264 data from configured input stream
   * 
   * Reads available H.264 data from the input stream (if configured) and
   * processes it through the decoder. Automatically handles NAL unit parsing
   * and frame assembly.
   * 
   * @return true if stream processing succeeded, false on error or no stream
   * 
   * @note Returns false if no input stream is configured
   * @note Amount of data read per call is limited by Config::max_read_bytes
   * @note Decoded frames trigger frame_callback if configured
   * @note Use this method in loop() for continuous stream processing
   */
  bool processStream() {
    ESP_LOGD(TAG, "processStream");
    if (!input_stream_) {
      ESP_LOGW(TAG, "No input stream configured");
      return false;
    }
 
    int available = input_stream_->available();
    if (available <= 0) return true;  // No data available, but not an error
 
    size_t bytes_to_read = cfg_.max_read_bytes > 0 ?
                          std::min((size_t)available, cfg_.max_read_bytes) :
                          (size_t)available;
 
    if (temp_buf_.size() < bytes_to_read) {
      temp_buf_.resize(bytes_to_read);
    }
 
    size_t bytes_read = input_stream_->readBytes(temp_buf_.data(), bytes_to_read);
    if (bytes_read > 0) {
      return decode(temp_buf_.data(), bytes_read);
    }
 
    return true;
  }
 
  /**
   * @brief Decode H.264 data from buffer
   * 
   * Processes H.264 encoded data directly from a memory buffer.
   * Handles NAL unit parsing, decoding, and output frame generation.
   * 
   * @param h264_data Pointer to H.264 encoded data buffer
   * @param h264_len Size of H.264 data in bytes
   * @return true if decoding succeeded, false on error
   * 
   * @note Input data should contain complete NAL units when possible
   * @note Decoder handles partial NAL units and reassembly internally
   * @note Decoded frames trigger frame_callback if configured
   * @note This method can be used without an input stream
   */
  bool decode(const uint8_t* h264_data, size_t h264_len) {
    ESP_LOGD(TAG, "decode");
    if (!dec_handle_ || !h264_data || h264_len == 0) return false;
 
    esp_h264_dec_in_frame_t in_frame{};
    esp_h264_dec_out_frame_t out_frame{};
 
    // Copy input data to internal buffer
    if (input_buf_.size() < h264_len) {
      input_buf_.resize(h264_len);
    }
    memcpy(input_buf_.data(), h264_data, h264_len);
 
    in_frame.raw_data.buffer = input_buf_.data();
    in_frame.raw_data.len = h264_len;
    in_frame.consume = 0;
 
    while (in_frame.raw_data.len > 0) {
      out_frame.outbuf = nullptr;
      out_frame.out_size = 0;
 
      esp_h264_err_t res = esp_h264_dec_process(dec_handle_, &in_frame, &out_frame);
 
      if (res != ESP_H264_ERR_OK) {
        decode_errors_++;
        ESP_LOGW(TAG, "Decode error: %d", (int)res);
        return false;
      }
 
      if (out_frame.out_size > 0 && out_frame.outbuf) {
        // Frame decoded successfully
        frame_ready_ = true;
        frame_count_++;
 
        // Get frame dimensions from decoder parameters
        esp_h264_dec_param_handle_t param_hd;
        esp_h264_dec_sw_get_param_hd(dec_handle_, &param_hd);
        esp_h264_resolution_t resolution;
        if (esp_h264_dec_get_resolution(param_hd, &resolution) == ESP_H264_ERR_OK) {
          frame_width_ = resolution.width;
          frame_height_ = resolution.height;
        }
 
        // Process decoded frame
        if (!processDecodedFrame(out_frame.outbuf, out_frame.out_size)) {
          return false;
        }
      }
 
      // Update input buffer for next iteration
      in_frame.raw_data.buffer += in_frame.consume;
      in_frame.raw_data.len -= in_frame.consume;
    }
 
    return true;
  }
 
  /**
   * @brief Check if a decoded frame is available
   * 
   * @return true if a decoded frame is ready for retrieval, false otherwise
   * 
   * @note Frame availability is reset after calling getFrame()
   * @note This method is useful for polling-based frame processing
   */
  bool hasFrame() const {
    ESP_LOGD(TAG, "hasFrame");
    return frame_ready_;
  }
 
  /**
   * @brief Retrieve decoded frame data
   * 
   * Copies the most recently decoded frame to the provided buffer.
   * Frame dimensions and format are returned via reference parameters.
   * 
   * @param dst Destination buffer for frame data
   * @param dst_len Size of destination buffer in bytes
   * @param width Reference to store frame width in pixels
   * @param height Reference to store frame height in pixels
   * @return true if frame copied successfully, false on error or no frame
   * 
   * @note Buffer must be large enough for the frame data
   * @note Frame availability is reset after successful retrieval
   * @note Frame format matches Config::output_format setting
   */
  bool getFrame(uint8_t* dst, size_t dst_len, uint32_t& width, uint32_t& height) {
    ESP_LOGD(TAG, "getFrame");
    if (!frame_ready_ || !dst) return false;
 
    size_t frame_size = frame_width_ * frame_height_ *
                       ESP_H264_GET_BPP_BY_PIC_TYPE(cfg_.output_format);
 
    if (dst_len < frame_size) {
      ESP_LOGE(TAG, "Destination buffer too small: need %u got %u",
               (unsigned)frame_size, (unsigned)dst_len);
      return false;
    }
 
    memcpy(dst, output_buf_.data(), frame_size);
    width = frame_width_;
    height = frame_height_;
    frame_ready_ = false;  // Reset frame availability
 
    return true;
  }
 
  /**
   * @brief Get total number of successfully decoded frames
   * 
   * @return Number of frames decoded since begin() was called
   * 
   * @note Counter is reset when begin() is called
   * @note Useful for monitoring decoder performance
   */
  uint32_t getFrameCount() const {
    ESP_LOGD(TAG, "getFrameCount");
    return frame_count_;
  }
 
  /**
   * @brief Get total number of decode errors
   * 
   * @return Number of decode errors since begin() was called
   * 
   * @note Counter is reset when begin() is called
   * @note Useful for monitoring stream quality and decoder health
   */
  uint32_t getDecodeErrors() const {
    ESP_LOGD(TAG, "getDecodeErrors");
    return decode_errors_;
  }
 
  /**
   * @brief Get current frame dimensions
   * 
   * @param width Reference to store frame width in pixels
   * @param height Reference to store frame height in pixels
   * @return true if valid dimensions available, false if no frame decoded yet
   * 
   * @note Dimensions are available after the first successful frame decode
   * @note Dimensions may change if stream resolution changes
   */
  bool getFrameDimensions(uint32_t& width, uint32_t& height) const {
    ESP_LOGD(TAG, "getFrameDimensions");
    if (frame_count_ == 0 || frame_width_ == 0 || frame_height_ == 0) return false;
    width = frame_width_;
    height = frame_height_;
    return true;
  }
 
  /**
   * @brief Cleanup and release decoder resources
   * 
   * Deinitializes the H.264 decoder and releases all allocated
   * resources. Should be called when decoding is complete or when
   * switching decoder configurations.
   * 
   * @note Safe to call multiple times
   * @note Required before calling begin() with different Config
   * @note Automatically called by destructor
   */
  void end() {
    ESP_LOGD(TAG, "end");
    if (dec_handle_) {
      esp_h264_dec_close(dec_handle_);
      esp_h264_dec_del(dec_handle_);
      dec_handle_ = nullptr;
    }
    if (!input_buf_.empty()) {
      input_buf_.clear();
    }
    if (!output_buf_.empty()) {
      output_buf_.clear();
    }
    if (!temp_buf_.empty()) {
      temp_buf_.clear();
    }
    frame_ready_ = false;
    frame_count_ = 0;
    decode_errors_ = 0;
    frame_width_ = 0;
    frame_height_ = 0;
  }
 
 private:
  Config cfg_;
  Stream* input_stream_;
  esp_h264_dec_handle_t dec_handle_ = nullptr;
  std::vector<uint8_t, Alloc> input_buf_;
  std::vector<uint8_t, Alloc> output_buf_;
  std::vector<uint8_t, Alloc> temp_buf_;  // For stream reading
 
  // Frame state
  uint32_t frame_width_ = 0;
  uint32_t frame_height_ = 0;
  bool frame_ready_ = false;
 
  // Statistics
  uint32_t frame_count_ = 0;
  uint32_t decode_errors_ = 0;
 
  /**
   * @brief Initialize H.264 software decoder with Config settings
   * @return true if decoder creation and opening successful, false on error
   * @note Creates decoder instance for constrained baseline H.264 profile
   * @note Uses esp_h264 software decoder (tinyH264)
   */
  bool initDecoder() {
    ESP_LOGD(TAG, "initDecoder");
    esp_h264_dec_cfg_sw_t dec_cfg{};
    dec_cfg.pic_type = cfg_.output_format;
 
    esp_h264_err_t res = esp_h264_dec_sw_new(&dec_cfg, &dec_handle_);
    if (res != ESP_H264_ERR_OK || !dec_handle_) {
      ESP_LOGE(TAG, "esp_h264_dec_sw_new failed: %d", (int)res);
      return false;
    }
 
    res = esp_h264_dec_open(dec_handle_);
    if (res != ESP_H264_ERR_OK) {
      ESP_LOGE(TAG, "esp_h264_dec_open failed: %d", (int)res);
      esp_h264_dec_del(dec_handle_);
      dec_handle_ = nullptr;
      return false;
    }
 
    return true;
  }
 
  /**
   * @brief Allocate input and output buffers using configured allocator
   * @return true if buffer allocation successful, false on error
   * @note Allocates buffers based on Config buffer size settings
   * @note Uses template allocator (Alloc) for memory management
   */
  bool initBuffers() {
    ESP_LOGD(TAG, "initBuffers");
    try {
      input_buf_.clear();
      output_buf_.clear();
      temp_buf_.clear();
 
      input_buf_.resize(cfg_.input_buffer_size);
      output_buf_.resize(cfg_.output_buffer_size);
      temp_buf_.resize(cfg_.max_read_bytes > 0 ? cfg_.max_read_bytes : 1500);
    } catch (const std::bad_alloc& e) {
      ESP_LOGE(TAG, "Failed to allocate buffers: %s", e.what());
      input_buf_.clear();
      output_buf_.clear();
      temp_buf_.clear();
      return false;
    }
    return true;
  }
 
  /**
   * @brief Process decoded frame data and handle format conversion if needed
   * 
   * @param frame_data Pointer to decoded frame data (I420 format from decoder)
   * @param frame_size Size of frame data in bytes
   * @return true if frame processing succeeded, false on error
   * 
   * @note Handles format conversion from I420 to configured output format
   * @note Calls user frame_callback if configured
   * @note Updates internal frame state and buffers
   */
  bool processDecodedFrame(const uint8_t* frame_data, uint32_t frame_size) {
    ESP_LOGD(TAG, "processDecodedFrame");
    if (!frame_data || frame_size == 0) return false;
 
    // Calculate required output buffer size
    size_t required_size = frame_width_ * frame_height_ *
                          ESP_H264_GET_BPP_BY_PIC_TYPE(cfg_.output_format);
 
    if (output_buf_.size() < required_size) {
      output_buf_.resize(required_size);
    }
 
    // Handle format conversion if needed
    if (cfg_.output_format == ESP_H264_RAW_FMT_I420) {
      // Direct copy for I420 output
      memcpy(output_buf_.data(), frame_data, std::min(frame_size, (uint32_t)output_buf_.size()));
    } else {
      // Format conversion needed (e.g., I420 to RGB565)
      if (!convertFormat(frame_data, frame_size)) {
        ESP_LOGE(TAG, "Format conversion failed");
        return false;
      }
    }
 
    // Call user callback if provided
    if (cfg_.frame_callback) {
      cfg_.frame_callback(output_buf_.data(), frame_width_, frame_height_, cfg_.output_format);
    }
 
    return true;
  }
 
  /**
   * @brief Convert decoded I420 frame to configured output format
   * 
   * @param i420_data Pointer to I420 frame data from decoder
   * @param i420_size Size of I420 data in bytes
   * @return true if conversion succeeded, false on error
   * 
   * @note Currently supports conversion to RGB565_LE
   * @note Additional format conversions can be added here
   * @note Writes converted data to output_buf_
   */
  bool convertFormat(const uint8_t* i420_data, uint32_t i420_size) {
    ESP_LOGD(TAG, "convertFormat");
    switch (cfg_.output_format) {
      case ESP_H264_RAW_FMT_RGB565_LE:
        return convertI420ToRGB565(i420_data, i420_size);
      case ESP_H264_RAW_FMT_I420:
        // Already handled in processDecodedFrame
        return true;
      default:
        ESP_LOGE(TAG, "Unsupported output format: %d", (int)cfg_.output_format);
        return false;
    }
  }
 
  /**
   * @brief Convert I420 frame to RGB565 format
   * 
   * @param i420_data Pointer to I420 frame data (Y + U + V planes)
   * @param i420_size Size of I420 data in bytes
   * @return true if conversion succeeded, false on error
   * 
   * @note I420 format: Y plane (width×height), U plane (width/2×height/2), V plane (width/2×height/2)
   * @note RGB565 format: 16-bit RGB (5-6-5 bit allocation)
   * @note Performs YUV to RGB color space conversion
   */
  bool convertI420ToRGB565(const uint8_t* i420_data, uint32_t i420_size) {
    ESP_LOGD(TAG, "convertI420ToRGB565");
    if (!i420_data || frame_width_ == 0 || frame_height_ == 0) return false;
 
    const uint8_t* y_plane = i420_data;
    const uint8_t* u_plane = y_plane + frame_width_ * frame_height_;
    const uint8_t* v_plane = u_plane + (frame_width_ / 2) * (frame_height_ / 2);
 
    uint16_t* rgb565_out = (uint16_t*)output_buf_.data();
 
    for (uint32_t y = 0; y < frame_height_; y++) {
      for (uint32_t x = 0; x < frame_width_; x++) {
        // Get Y, U, V values
        uint8_t Y = y_plane[y * frame_width_ + x];
        uint8_t U = u_plane[(y / 2) * (frame_width_ / 2) + (x / 2)];
        uint8_t V = v_plane[(y / 2) * (frame_width_ / 2) + (x / 2)];
 
        // YUV to RGB conversion (ITU-R BT.601)
        int C = Y - 16;
        int D = U - 128;
        int E = V - 128;
 
        int R = (298 * C + 409 * E + 128) >> 8;
        int G = (298 * C - 100 * D - 208 * E + 128) >> 8;
        int B = (298 * C + 516 * D + 128) >> 8;
 
        // Clamp to 0-255
        R = std::max(0, std::min(255, R));
        G = std::max(0, std::min(255, G));
        B = std::max(0, std::min(255, B));
 
        // Convert to RGB565 (5-6-5 bits)
        uint16_t rgb565 = ((R & 0xF8) << 8) | ((G & 0xFC) << 3) | ((B & 0xF8) >> 3);
        rgb565_out[y * frame_width_ + x] = rgb565;
      }
    }
 
    return true;
  }
};
 
/**
 * @brief Type alias for H264Decoder using default PSRAM allocation
 * 
 * Provides a simpler way to declare H264Decoder instances without specifying
 * the allocator template parameter. Uses PSRAMAllocatorH264<uint8_t> by default.
 * 
 * Example usage:
 * @code
 * H264DecoderPSRAM decoder(&udp);  // Instead of H264Decoder<PSRAMAllocatorH264<uint8_t>>
 * @endcode
 */
using H264DecoderPSRAM = H264Decoder<PSRAMAllocatorH264<uint8_t>>;
 
/**
 * @brief Type alias for H264Decoder using internal RAM allocation
 * 
 * Provides a simpler way to declare H264Decoder instances that use fast
 * internal RAM instead of external PSRAM.
 * 
 * Example usage:
 * @code
 * H264DecoderRAM decoder(&udp);   // Instead of H264Decoder<RAMAllocatorH264<uint8_t>>
 * @endcode
 */
using H264DecoderRAM = H264Decoder<RAMAllocatorH264<uint8_t>>;
 
}  // namespace esp_h264
 
#ifdef ARDUINO
using namespace esp_h264;
#endif