动作识别 视频理解大模型

AI视觉网奇2026-08-15 10:13

目录

Qwen3-VL-32B-Instruct

Qwen/Qwen3-VL-8B-Instruct

[2. InternVideo2.5 ------ 目前非常成熟](#2. InternVideo2.5 —— 目前非常成熟)

InternVL2.5-26B-Instruct

yanziang/InternVideo3-8B-Instruct

如果是在你前面这个**"视频动作识别 / 羽毛球视频分析"**场景里比较：

Qwen3-VL-32B-Instruct

Qwen3-VL-32B-Instruct fp8 需要48g显存

Qwen/Qwen3-VL-8B-Instruct

2. InternVideo2.5 ------ 目前非常成熟

InternVL2.5-26B-Instruct

30G显存

hf download OpenGVLab/InternVL2_5-26B-AWQ --local-dir ./OpenGVLab/InternVL2_5-26B-AWQ

yanziang/InternVideo3-8B-Instruct

python 复制代码 
import torch
from transformers import AutoModelForCausalLM, AutoProcessor

model_path = "OpenGVLab/InternVideo3-8B-Instruct"
model_path = "/data/feature/lbg/models/yanziang_InternVideo3-8B-Instruct/"

model = AutoModelForCausalLM.from_pretrained(
    model_path,
    dtype=torch.bfloat16,
    attn_implementation="sdpa",
    device_map="auto",
    trust_remote_code=True,
)

processor = AutoProcessor.from_pretrained(
    model_path,
    trust_remote_code=True,
)

video_path = "/data/lbg/project/aigc/manim-generator/yumao_v/yq_highlight_1785620340669_124.mp4"

max_frames=128
min_frames=16
fps = 1
min_pixels = 128 * 32 * 32
max_pixels = 128 * 32 * 32

messages = [
    {
        "role": "user",
        "content": [
            {"type": "video", "video": video_path, "fps": fps},
            {"type": "text", "text": "Please describe this video in detail."},
        ],
    }
]

processor.video_processor.size = {
    "longest_edge": max_pixels * max_frames,
    "shortest_edge": min_pixels * min_frames,
}

inputs = processor.apply_chat_template(
    messages,
    tokenize=True,
    add_generation_prompt=True,
    return_dict=True,
    fps=fps,
    return_tensors="pt",
)
inputs = inputs.to(model.device)

output = model.generate(**inputs, max_new_tokens=1024, use_cache=True)
generated_ids = [o[len(i):] for i, o in zip(inputs.input_ids, output)]
print(processor.batch_decode(generated_ids, skip_special_tokens=True)[0])

封装server：

python 复制代码 
# main.py
import torch
from transformers import AutoModelForCausalLM, AutoProcessor
from fastapi import FastAPI, HTTPException, UploadFile, File, Form
from fastapi.responses import JSONResponse
from pydantic import BaseModel
import tempfile
import os
import shutil
from typing import Optional, List
import base64
from io import BytesIO
from PIL import Image
import logging
from contextlib import asynccontextmanager

# 配置日志
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)

# 全局模型变量
model = None
processor = None
model_loaded = False

# 配置
MODEL_PATH = "/data/feature/lbg/models/yanziang_InternVideo3-8B-Instruct/"
MAX_FRAMES = 128
MIN_FRAMES = 16
MIN_PIXELS = 128 * 32 * 32
MAX_PIXELS = 128 * 32 * 32
MAX_NEW_TOKENS = 1024

# 请求模型
class VideoRequest(BaseModel):
    video_path: str
    fps: int = 1
    max_new_tokens: int = MAX_NEW_TOKENS
    prompt: str = "Please describe this video in detail."
    max_frames: Optional[int] = MAX_FRAMES
    min_frames: Optional[int] = MIN_FRAMES

class ImageRequest(BaseModel):
    image_path: Optional[str] = None
    prompt: str = "Please describe this image in detail."
    max_new_tokens: int = MAX_NEW_TOKENS

class ChatRequest(BaseModel):
    messages: List[dict]
    max_new_tokens: int = MAX_NEW_TOKENS
    fps: int = 1

# 响应模型
class InferenceResponse(BaseModel):
    success: bool
    result: Optional[str] = None
    error: Optional[str] = None

def load_model():
    """加载模型和处理器"""
    global model, processor, model_loaded
    
    if model_loaded:
        logger.info("Model already loaded")
        return
    
    try:
        logger.info(f"Loading model from {MODEL_PATH}")
        
        model = AutoModelForCausalLM.from_pretrained(
            MODEL_PATH,
            torch_dtype=torch.bfloat16,
            attn_implementation="sdpa",
            device_map="auto",
            trust_remote_code=True,
        )
        
        processor = AutoProcessor.from_pretrained(
            MODEL_PATH,
            trust_remote_code=True,
        )
        
        model_loaded = True
        logger.info("Model loaded successfully")
        
    except Exception as e:
        logger.error(f"Failed to load model: {str(e)}")
        raise  # 启动时加载失败，让应用启动失败

def unload_model():
    """卸载模型，释放资源"""
    global model, processor, model_loaded
    if model_loaded:
        logger.info("Unloading model...")
        model = None
        processor = None
        model_loaded = False
        # 清理GPU缓存
        if torch.cuda.is_available():
            torch.cuda.empty_cache()
        logger.info("Model unloaded")

# --- 使用 lifespan 替代 on_event ---
@asynccontextmanager
async def lifespan(app: FastAPI):
    # 启动逻辑
    logger.info("Application startup: Loading model...")
    load_model()
    yield
    # 关闭逻辑
    logger.info("Application shutdown: Cleaning up resources...")
    unload_model()

# --- 创建 FastAPI 应用，传入 lifespan 参数 ---
app = FastAPI(
    title="InternVideo3 API",
    description="Video and Image Understanding API",
    lifespan=lifespan
)

@app.get("/health")
async def health_check():
    """健康检查"""
    return {"status": "healthy", "model_loaded": model_loaded}

@app.post("/predict/video_path", response_model=InferenceResponse)
async def predict_video_path(request: VideoRequest):
    """
    使用视频路径进行推理
    """
    if not model_loaded:
        raise HTTPException(status_code=503, detail="Model not loaded")
    
    try:
        # 准备消息
        messages = [
            {
                "role": "user",
                "content": [
                    {"type": "video", "video": request.video_path, "fps": request.fps},
                    {"type": "text", "text": request.prompt},
                ],
            }
        ]
        
        # 设置视频处理器参数
        processor.video_processor.size = {
            "longest_edge": MAX_PIXELS * request.max_frames,
            "shortest_edge": MIN_PIXELS * request.min_frames,
        }
        
        # 处理输入
        inputs = processor.apply_chat_template(
            messages,
            tokenize=True,
            add_generation_prompt=True,
            return_dict=True,
            fps=request.fps,
            return_tensors="pt",
        )
        inputs = inputs.to(model.device)
        
        # 生成输出
        output = model.generate(**inputs, max_new_tokens=request.max_new_tokens, use_cache=True)
        generated_ids = [o[len(i):] for i, o in zip(inputs.input_ids, output)]
        result = processor.batch_decode(generated_ids, skip_special_tokens=True)[0]
        
        return InferenceResponse(success=True, result=result)
        
    except Exception as e:
        logger.error(f"Prediction error: {str(e)}")
        return InferenceResponse(success=False, error=str(e))

@app.post("/predict/video_upload", response_model=InferenceResponse)
async def predict_video_upload(
    file: UploadFile = File(...),
    prompt: str = Form("Please describe this video in detail."),
    fps: int = Form(1),
    max_new_tokens: int = Form(MAX_NEW_TOKENS)
):
    """
    上传视频文件进行推理
    """
    if not model_loaded:
        raise HTTPException(status_code=503, detail="Model not loaded")
    
    temp_file_path = None
    
    try:
        # 创建临时文件
        with tempfile.NamedTemporaryFile(delete=False, suffix=".mp4") as temp_file:
            shutil.copyfileobj(file.file, temp_file)
            temp_file_path = temp_file.name
        
        # 准备消息
        messages = [
            {
                "role": "user",
                "content": [
                    {"type": "video", "video": temp_file_path, "fps": fps},
                    {"type": "text", "text": prompt},
                ],
            }
        ]
        
        # 设置视频处理器参数
        processor.video_processor.size = {
            "longest_edge": MAX_PIXELS * MAX_FRAMES,
            "shortest_edge": MIN_PIXELS * MIN_FRAMES,
        }
        
        # 处理输入
        inputs = processor.apply_chat_template(
            messages,
            tokenize=True,
            add_generation_prompt=True,
            return_dict=True,
            fps=fps,
            return_tensors="pt",
        )
        inputs = inputs.to(model.device)
        
        # 生成输出
        output = model.generate(**inputs, max_new_tokens=max_new_tokens, use_cache=True)
        generated_ids = [o[len(i):] for i, o in zip(inputs.input_ids, output)]
        result = processor.batch_decode(generated_ids, skip_special_tokens=True)[0]
        
        return InferenceResponse(success=True, result=result)
        
    except Exception as e:
        logger.error(f"Prediction error: {str(e)}")
        return InferenceResponse(success=False, error=str(e))
        
    finally:
        # 清理临时文件
        if temp_file_path and os.path.exists(temp_file_path):
            os.unlink(temp_file_path)
        if file.file:
            file.file.close()

@app.post("/predict/image_path", response_model=InferenceResponse)
async def predict_image_path(request: ImageRequest):
    """
    使用图像路径进行推理
    """
    if not model_loaded:
        raise HTTPException(status_code=503, detail="Model not loaded")
    
    try:
        # 准备消息
        messages = [
            {
                "role": "user",
                "content": [
                    {"type": "image", "image": request.image_path},
                    {"type": "text", "text": request.prompt},
                ],
            }
        ]
        
        # 处理输入
        inputs = processor.apply_chat_template(
            messages,
            tokenize=True,
            add_generation_prompt=True,
            return_dict=True,
            return_tensors="pt",
        )
        inputs = inputs.to(model.device)
        
        # 生成输出
        output = model.generate(**inputs, max_new_tokens=request.max_new_tokens, use_cache=True)
        generated_ids = [o[len(i):] for i, o in zip(inputs.input_ids, output)]
        result = processor.batch_decode(generated_ids, skip_special_tokens=True)[0]
        
        return InferenceResponse(success=True, result=result)
        
    except Exception as e:
        logger.error(f"Prediction error: {str(e)}")
        return InferenceResponse(success=False, error=str(e))

@app.post("/predict/image_upload", response_model=InferenceResponse)
async def predict_image_upload(
    file: UploadFile = File(...),
    prompt: str = Form("Please describe this image in detail."),
    max_new_tokens: int = Form(MAX_NEW_TOKENS)
):
    """
    上传图像文件进行推理
    """
    if not model_loaded:
        raise HTTPException(status_code=503, detail="Model not loaded")
    
    temp_file_path = None
    
    try:
        # 创建临时文件
        file_extension = os.path.splitext(file.filename)[1]
        with tempfile.NamedTemporaryFile(delete=False, suffix=file_extension) as temp_file:
            shutil.copyfileobj(file.file, temp_file)
            temp_file_path = temp_file.name
        
        # 准备消息
        messages = [
            {
                "role": "user",
                "content": [
                    {"type": "image", "image": temp_file_path},
                    {"type": "text", "text": prompt},
                ],
            }
        ]
        
        # 处理输入
        inputs = processor.apply_chat_template(
            messages,
            tokenize=True,
            add_generation_prompt=True,
            return_dict=True,
            return_tensors="pt",
        )
        inputs = inputs.to(model.device)
        
        # 生成输出
        output = model.generate(**inputs, max_new_tokens=max_new_tokens, use_cache=True)
        generated_ids = [o[len(i):] for i, o in zip(inputs.input_ids, output)]
        result = processor.batch_decode(generated_ids, skip_special_tokens=True)[0]
        
        return InferenceResponse(success=True, result=result)
        
    except Exception as e:
        logger.error(f"Prediction error: {str(e)}")
        return InferenceResponse(success=False, error=str(e))
        
    finally:
        # 清理临时文件
        if temp_file_path and os.path.exists(temp_file_path):
            os.unlink(temp_file_path)
        if file.file:
            file.file.close()

@app.post("/predict/chat", response_model=InferenceResponse)
async def predict_chat(request: ChatRequest):
    """
    通用对话接口，支持多种输入类型
    """
    if not model_loaded:
        raise HTTPException(status_code=503, detail="Model not loaded")
    
    try:
        # 处理输入
        inputs = processor.apply_chat_template(
            request.messages,
            tokenize=True,
            add_generation_prompt=True,
            return_dict=True,
            fps=request.fps,
            return_tensors="pt",
        )
        inputs = inputs.to(model.device)
        
        # 生成输出
        output = model.generate(**inputs, max_new_tokens=request.max_new_tokens, use_cache=True)
        generated_ids = [o[len(i):] for i, o in zip(inputs.input_ids, output)]
        result = processor.batch_decode(generated_ids, skip_special_tokens=True)[0]
        
        return InferenceResponse(success=True, result=result)
        
    except Exception as e:
        logger.error(f"Chat prediction error: {str(e)}")
        return InferenceResponse(success=False, error=str(e))

if __name__ == "__main__":
    import uvicorn
    uvicorn.run(
        "qianwen_server:app",
        host="0.0.0.0",
        port=7999,
        reload=False,
        workers=1,  # 由于模型占用大量内存，建议只使用1个worker
    )
相关推荐
Sylvia33.17 分钟前
LOL实时数据接入深度解析：从WebSocket到完整数据模型
java·网络·python·websocket·网络协议
梦想不只是梦与想18 分钟前
Web 服务器网关协议：ASGI 和 WSGI
服务器·python·fastapi
ai小陈3 小时前
PyTorch DataLoader数据加载性能排查：GPU利用率低的实操指南
人工智能·pytorch·python·深度学习·ai·gpu算力
2601_962299245 小时前
Azure python操作系统列表
python·操作系统·azure·虚拟机·存储配置文件
2601_962071576 小时前
【Java报错已解决】org.springframework.beans.factory.BeanCreationException
java·开发语言
学术 学术 Fun6 小时前
如何用 API 与 Webhook 批量把图表图片转换成 VSDX
python·自动化·api·visio
m0_614523556 小时前
完整教程｜输入一句描述，能不能直接生成一段可以继续剪辑的视频：写结构化描述到生成短样片
人工智能·音视频
淡海水7 小时前
07-04-并发-ConcurrentBag-T-工作窃取WorkStealing算法
开发语言·算法·c#·bag·concurrent·workstealing
2601_962100547 小时前
python包和模块的内容整理
python·模块·导入·虚拟环境·包
CS_Zero7 小时前
C语言sizeof是函数吗？
c语言·开发语言
热门推荐
01国内可直接用、免费额度/永久免费的大模型API清单（含 SiliconFlow、火山、阿里、智谱、百度、Kimi、DeepSeek、DMXAPI 等）02GitHub 镜像站点03ZCode 周末送额度活动开启：3 亿 Token 免费领取04历年考研数学一、数学二、数学三真题试卷及答案PDF05蜘蛛磁力 搜索引擎大全，如何使用蜘蛛磁力查找磁力链接062026 国产 AI 大模型横评：DeepSeek、通义千问、Kimi、文心一言、星火、豆包谁更能打？07微信历史版本含下载地址（ Windows PC | 安卓 | MAC ）及设置微信不更新08AI 编程 IDE 全景解析 2026：Agent 全面接管开发链路09Agnes AI 免费 API 接入指南：文本、生图、生视频，一套接口全免费10VSCode+CodeX扩展，无法启动