动作识别 视频理解大模型

AI视觉网奇2026-08-15 10:13

目录

Qwen3-VL-32B-Instruct

Qwen/Qwen3-VL-8B-Instruct

[2. InternVideo2.5 ------ 目前非常成熟](#2. InternVideo2.5 —— 目前非常成熟)

InternVL2.5-26B-Instruct

yanziang/InternVideo3-8B-Instruct

如果是在你前面这个**"视频动作识别 / 羽毛球视频分析"**场景里比较：

Qwen3-VL-32B-Instruct

Qwen3-VL-32B-Instruct fp8 需要48g显存

Qwen/Qwen3-VL-8B-Instruct

2. InternVideo2.5 ------ 目前非常成熟

InternVL2.5-26B-Instruct

30G显存

hf download OpenGVLab/InternVL2_5-26B-AWQ --local-dir ./OpenGVLab/InternVL2_5-26B-AWQ

yanziang/InternVideo3-8B-Instruct

python 复制代码 
import torch
from transformers import AutoModelForCausalLM, AutoProcessor

model_path = "OpenGVLab/InternVideo3-8B-Instruct"
model_path = "/data/feature/lbg/models/yanziang_InternVideo3-8B-Instruct/"

model = AutoModelForCausalLM.from_pretrained(
    model_path,
    dtype=torch.bfloat16,
    attn_implementation="sdpa",
    device_map="auto",
    trust_remote_code=True,
)

processor = AutoProcessor.from_pretrained(
    model_path,
    trust_remote_code=True,
)

video_path = "/data/lbg/project/aigc/manim-generator/yumao_v/yq_highlight_1785620340669_124.mp4"

max_frames=128
min_frames=16
fps = 1
min_pixels = 128 * 32 * 32
max_pixels = 128 * 32 * 32

messages = [
    {
        "role": "user",
        "content": [
            {"type": "video", "video": video_path, "fps": fps},
            {"type": "text", "text": "Please describe this video in detail."},
        ],
    }
]

processor.video_processor.size = {
    "longest_edge": max_pixels * max_frames,
    "shortest_edge": min_pixels * min_frames,
}

inputs = processor.apply_chat_template(
    messages,
    tokenize=True,
    add_generation_prompt=True,
    return_dict=True,
    fps=fps,
    return_tensors="pt",
)
inputs = inputs.to(model.device)

output = model.generate(**inputs, max_new_tokens=1024, use_cache=True)
generated_ids = [o[len(i):] for i, o in zip(inputs.input_ids, output)]
print(processor.batch_decode(generated_ids, skip_special_tokens=True)[0])

封装server：

python 复制代码 
# main.py
import torch
from transformers import AutoModelForCausalLM, AutoProcessor
from fastapi import FastAPI, HTTPException, UploadFile, File, Form
from fastapi.responses import JSONResponse
from pydantic import BaseModel
import tempfile
import os
import shutil
from typing import Optional, List
import base64
from io import BytesIO
from PIL import Image
import logging
from contextlib import asynccontextmanager

# 配置日志
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)

# 全局模型变量
model = None
processor = None
model_loaded = False

# 配置
MODEL_PATH = "/data/feature/lbg/models/yanziang_InternVideo3-8B-Instruct/"
MAX_FRAMES = 128
MIN_FRAMES = 16
MIN_PIXELS = 128 * 32 * 32
MAX_PIXELS = 128 * 32 * 32
MAX_NEW_TOKENS = 1024

# 请求模型
class VideoRequest(BaseModel):
    video_path: str
    fps: int = 1
    max_new_tokens: int = MAX_NEW_TOKENS
    prompt: str = "Please describe this video in detail."
    max_frames: Optional[int] = MAX_FRAMES
    min_frames: Optional[int] = MIN_FRAMES

class ImageRequest(BaseModel):
    image_path: Optional[str] = None
    prompt: str = "Please describe this image in detail."
    max_new_tokens: int = MAX_NEW_TOKENS

class ChatRequest(BaseModel):
    messages: List[dict]
    max_new_tokens: int = MAX_NEW_TOKENS
    fps: int = 1

# 响应模型
class InferenceResponse(BaseModel):
    success: bool
    result: Optional[str] = None
    error: Optional[str] = None

def load_model():
    """加载模型和处理器"""
    global model, processor, model_loaded
    
    if model_loaded:
        logger.info("Model already loaded")
        return
    
    try:
        logger.info(f"Loading model from {MODEL_PATH}")
        
        model = AutoModelForCausalLM.from_pretrained(
            MODEL_PATH,
            torch_dtype=torch.bfloat16,
            attn_implementation="sdpa",
            device_map="auto",
            trust_remote_code=True,
        )
        
        processor = AutoProcessor.from_pretrained(
            MODEL_PATH,
            trust_remote_code=True,
        )
        
        model_loaded = True
        logger.info("Model loaded successfully")
        
    except Exception as e:
        logger.error(f"Failed to load model: {str(e)}")
        raise  # 启动时加载失败，让应用启动失败

def unload_model():
    """卸载模型，释放资源"""
    global model, processor, model_loaded
    if model_loaded:
        logger.info("Unloading model...")
        model = None
        processor = None
        model_loaded = False
        # 清理GPU缓存
        if torch.cuda.is_available():
            torch.cuda.empty_cache()
        logger.info("Model unloaded")

# --- 使用 lifespan 替代 on_event ---
@asynccontextmanager
async def lifespan(app: FastAPI):
    # 启动逻辑
    logger.info("Application startup: Loading model...")
    load_model()
    yield
    # 关闭逻辑
    logger.info("Application shutdown: Cleaning up resources...")
    unload_model()

# --- 创建 FastAPI 应用，传入 lifespan 参数 ---
app = FastAPI(
    title="InternVideo3 API",
    description="Video and Image Understanding API",
    lifespan=lifespan
)

@app.get("/health")
async def health_check():
    """健康检查"""
    return {"status": "healthy", "model_loaded": model_loaded}

@app.post("/predict/video_path", response_model=InferenceResponse)
async def predict_video_path(request: VideoRequest):
    """
    使用视频路径进行推理
    """
    if not model_loaded:
        raise HTTPException(status_code=503, detail="Model not loaded")
    
    try:
        # 准备消息
        messages = [
            {
                "role": "user",
                "content": [
                    {"type": "video", "video": request.video_path, "fps": request.fps},
                    {"type": "text", "text": request.prompt},
                ],
            }
        ]
        
        # 设置视频处理器参数
        processor.video_processor.size = {
            "longest_edge": MAX_PIXELS * request.max_frames,
            "shortest_edge": MIN_PIXELS * request.min_frames,
        }
        
        # 处理输入
        inputs = processor.apply_chat_template(
            messages,
            tokenize=True,
            add_generation_prompt=True,
            return_dict=True,
            fps=request.fps,
            return_tensors="pt",
        )
        inputs = inputs.to(model.device)
        
        # 生成输出
        output = model.generate(**inputs, max_new_tokens=request.max_new_tokens, use_cache=True)
        generated_ids = [o[len(i):] for i, o in zip(inputs.input_ids, output)]
        result = processor.batch_decode(generated_ids, skip_special_tokens=True)[0]
        
        return InferenceResponse(success=True, result=result)
        
    except Exception as e:
        logger.error(f"Prediction error: {str(e)}")
        return InferenceResponse(success=False, error=str(e))

@app.post("/predict/video_upload", response_model=InferenceResponse)
async def predict_video_upload(
    file: UploadFile = File(...),
    prompt: str = Form("Please describe this video in detail."),
    fps: int = Form(1),
    max_new_tokens: int = Form(MAX_NEW_TOKENS)
):
    """
    上传视频文件进行推理
    """
    if not model_loaded:
        raise HTTPException(status_code=503, detail="Model not loaded")
    
    temp_file_path = None
    
    try:
        # 创建临时文件
        with tempfile.NamedTemporaryFile(delete=False, suffix=".mp4") as temp_file:
            shutil.copyfileobj(file.file, temp_file)
            temp_file_path = temp_file.name
        
        # 准备消息
        messages = [
            {
                "role": "user",
                "content": [
                    {"type": "video", "video": temp_file_path, "fps": fps},
                    {"type": "text", "text": prompt},
                ],
            }
        ]
        
        # 设置视频处理器参数
        processor.video_processor.size = {
            "longest_edge": MAX_PIXELS * MAX_FRAMES,
            "shortest_edge": MIN_PIXELS * MIN_FRAMES,
        }
        
        # 处理输入
        inputs = processor.apply_chat_template(
            messages,
            tokenize=True,
            add_generation_prompt=True,
            return_dict=True,
            fps=fps,
            return_tensors="pt",
        )
        inputs = inputs.to(model.device)
        
        # 生成输出
        output = model.generate(**inputs, max_new_tokens=max_new_tokens, use_cache=True)
        generated_ids = [o[len(i):] for i, o in zip(inputs.input_ids, output)]
        result = processor.batch_decode(generated_ids, skip_special_tokens=True)[0]
        
        return InferenceResponse(success=True, result=result)
        
    except Exception as e:
        logger.error(f"Prediction error: {str(e)}")
        return InferenceResponse(success=False, error=str(e))
        
    finally:
        # 清理临时文件
        if temp_file_path and os.path.exists(temp_file_path):
            os.unlink(temp_file_path)
        if file.file:
            file.file.close()

@app.post("/predict/image_path", response_model=InferenceResponse)
async def predict_image_path(request: ImageRequest):
    """
    使用图像路径进行推理
    """
    if not model_loaded:
        raise HTTPException(status_code=503, detail="Model not loaded")
    
    try:
        # 准备消息
        messages = [
            {
                "role": "user",
                "content": [
                    {"type": "image", "image": request.image_path},
                    {"type": "text", "text": request.prompt},
                ],
            }
        ]
        
        # 处理输入
        inputs = processor.apply_chat_template(
            messages,
            tokenize=True,
            add_generation_prompt=True,
            return_dict=True,
            return_tensors="pt",
        )
        inputs = inputs.to(model.device)
        
        # 生成输出
        output = model.generate(**inputs, max_new_tokens=request.max_new_tokens, use_cache=True)
        generated_ids = [o[len(i):] for i, o in zip(inputs.input_ids, output)]
        result = processor.batch_decode(generated_ids, skip_special_tokens=True)[0]
        
        return InferenceResponse(success=True, result=result)
        
    except Exception as e:
        logger.error(f"Prediction error: {str(e)}")
        return InferenceResponse(success=False, error=str(e))

@app.post("/predict/image_upload", response_model=InferenceResponse)
async def predict_image_upload(
    file: UploadFile = File(...),
    prompt: str = Form("Please describe this image in detail."),
    max_new_tokens: int = Form(MAX_NEW_TOKENS)
):
    """
    上传图像文件进行推理
    """
    if not model_loaded:
        raise HTTPException(status_code=503, detail="Model not loaded")
    
    temp_file_path = None
    
    try:
        # 创建临时文件
        file_extension = os.path.splitext(file.filename)[1]
        with tempfile.NamedTemporaryFile(delete=False, suffix=file_extension) as temp_file:
            shutil.copyfileobj(file.file, temp_file)
            temp_file_path = temp_file.name
        
        # 准备消息
        messages = [
            {
                "role": "user",
                "content": [
                    {"type": "image", "image": temp_file_path},
                    {"type": "text", "text": prompt},
                ],
            }
        ]
        
        # 处理输入
        inputs = processor.apply_chat_template(
            messages,
            tokenize=True,
            add_generation_prompt=True,
            return_dict=True,
            return_tensors="pt",
        )
        inputs = inputs.to(model.device)
        
        # 生成输出
        output = model.generate(**inputs, max_new_tokens=max_new_tokens, use_cache=True)
        generated_ids = [o[len(i):] for i, o in zip(inputs.input_ids, output)]
        result = processor.batch_decode(generated_ids, skip_special_tokens=True)[0]
        
        return InferenceResponse(success=True, result=result)
        
    except Exception as e:
        logger.error(f"Prediction error: {str(e)}")
        return InferenceResponse(success=False, error=str(e))
        
    finally:
        # 清理临时文件
        if temp_file_path and os.path.exists(temp_file_path):
            os.unlink(temp_file_path)
        if file.file:
            file.file.close()

@app.post("/predict/chat", response_model=InferenceResponse)
async def predict_chat(request: ChatRequest):
    """
    通用对话接口，支持多种输入类型
    """
    if not model_loaded:
        raise HTTPException(status_code=503, detail="Model not loaded")
    
    try:
        # 处理输入
        inputs = processor.apply_chat_template(
            request.messages,
            tokenize=True,
            add_generation_prompt=True,
            return_dict=True,
            fps=request.fps,
            return_tensors="pt",
        )
        inputs = inputs.to(model.device)
        
        # 生成输出
        output = model.generate(**inputs, max_new_tokens=request.max_new_tokens, use_cache=True)
        generated_ids = [o[len(i):] for i, o in zip(inputs.input_ids, output)]
        result = processor.batch_decode(generated_ids, skip_special_tokens=True)[0]
        
        return InferenceResponse(success=True, result=result)
        
    except Exception as e:
        logger.error(f"Chat prediction error: {str(e)}")
        return InferenceResponse(success=False, error=str(e))

if __name__ == "__main__":
    import uvicorn
    uvicorn.run(
        "qianwen_server:app",
        host="0.0.0.0",
        port=7999,
        reload=False,
        workers=1,  # 由于模型占用大量内存，建议只使用1个worker
    )
相关推荐
默_笙3 天前
🍙 给每个请求过安检：FastAPI 是怎么把校验写进类型注解的
python
小羊没烦恼！3 天前
初探性能优化——2个月到4小时的性能提升
java·开发语言·windows·算法·c#
qq_426003963 天前
启动playwright录制codegen生成自动化测试脚本
python·自动化
虎头金猫3 天前
4K 视频总卡在公网带宽？用 N1 + OpenList 把网盘播放链路重新理顺
运维·服务器·网络·python·容器·beautifulsoup·pandas
美狐美颜SDK开放平台3 天前
开发直播APP时如何接入视频美颜SDK？开发流程与注意事项
android·人工智能·计算机视觉·音视频·直播美颜sdk
长沙三为智能科技3 天前
家政小程序开发从0到上线：五阶段交付流程与验收清单
python
伞伞悦读3 天前
【第38期】Python 模块与包详解：import、from、模块搜索路径、包结构和 __init__
开发语言·python
只睡四小时3 天前
Canvas 弹道联机实战：700 行 + 固定时间步长
python·websocket·html5·游戏开发·canvas
C语言小火车3 天前
C/C++ 为什么需要编译器？
开发语言·c++
奇思妙想聪明勤奋的小羊3 天前
DeepAgents第5章：子Agent 与上下文隔离—让 Agent学会委派
人工智能·python·学习·语言模型
热门推荐
01国内可直接用、免费额度/永久免费的大模型API清单（含 SiliconFlow、火山、阿里、智谱、百度、Kimi、DeepSeek、DMXAPI 等）02Telegram APK 下载安装完整指南 — 2026年最新03GitHub 镜像站点04蜘蛛磁力 搜索引擎大全，如何使用蜘蛛磁力查找磁力链接05历年考研数学一、数学二、数学三真题试卷及答案PDF06如何新建文件夹？ 电脑新建文件夹的4种方法072026 爱知・名古屋亚运会怎么看？直播平台+赛程安排分享08微信历史版本含下载地址（ Windows PC | 安卓 | MAC ）及设置微信不更新09人工智能（AI）发展史：详细里程碑102026年CSP-J初赛分数线-----预测