记录一个`ffmpeg`的`swscale`库crash的例子

记录一个ffmpeg的swscale库crash的例子

本质上, 就是simd越界导致, 具体场景如下:

外部api分配了rgb24内存指针ptr, 然后转换为yuv420p, 如果ptr的end不可访问, 则会crash

1. simd越界例子

#include <cstdint>
#include <immintrin.h>
#include <iostream>
#include <memory>

#ifdef _WIN32
#include <windows.h>
#elif __linux__
#include <sys/mman.h>
#include <unistd.h>
#endif

#ifdef _WIN32
void win_crash() {
  constexpr size_t N = 1024;
  constexpr size_t PAGE = 4096;
  std::shared_ptr<uint8_t> base(
      static_cast<uint8_t *>(VirtualAlloc(
          nullptr, PAGE * 2, MEM_COMMIT | MEM_RESERVE, PAGE_READWRITE)),
      [](uint8_t *p) { VirtualFree(p, 0, MEM_RELEASE); });
  DWORD old;
  VirtualProtect(base.get() + PAGE, PAGE, PAGE_NOACCESS, &old);
  uint8_t *data = base.get() + PAGE - N;
  for (size_t i = 0; i < N; ++i)
    data[i] = static_cast<uint8_t>(i);

  for (size_t i = 0; i < N; i += 8) {
    __m256i v =
        _mm256_loadu_si256(reinterpret_cast<const __m256i *>(data + 32 + i));
    uint8_t tmp = static_cast<uint8_t>(_mm256_extract_epi8(v, 0));
    std::cout << static_cast<int64_t>(tmp) << "\n";
  }
}
#endif

#ifdef __linux__
// target_compile_options(my_target_name PRIVATE -mavx2)
void linux_crash() {
  constexpr size_t N = 1024;
  constexpr size_t PAGE = 4096;
  void *p = mmap(nullptr, PAGE * 2, PROT_READ | PROT_WRITE,
                 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
  if (p == MAP_FAILED) {
    perror("mmap");
    return;
  }
  std::shared_ptr<uint8_t> base(static_cast<uint8_t *>(p),
                                [](uint8_t *p) { munmap(p, PAGE * 2); });
  if (mprotect(base.get() + PAGE, PAGE, PROT_NONE) != 0) {
    perror("mprotect");
    return;
  }
  uint8_t *data = base.get() + PAGE - N;
  for (size_t i = 0; i < N; ++i)
    data[i] = static_cast<uint8_t>(i);
  for (size_t i = 0; i < N; i += 8) {
    __m256i v = _mm256_loadu_si256(reinterpret_cast<const __m256i *>(data + i));
    uint8_t tmp = static_cast<uint8_t>(_mm256_extract_epi8(v, 0));
    std::cout << static_cast<int64_t>(tmp) << "\n";
  }
}
#endif

int main() {
  win_crash();
  // linux_crash();
  return 0;
}

ffmpeg堆栈错误信息

最终崩溃在函数指针lumToYV12中, 具体汇编代码见libswscale/x86/swscale.c和libswscale/input.asm

reference

• ffmpeg笔记