目录
[2. InternVideo2.5 ------ 目前非常成熟](#2. InternVideo2.5 —— 目前非常成熟)
yanziang/InternVideo3-8B-Instruct
如果是在你前面这个**"视频动作识别 / 羽毛球视频分析"**场景里比较：
Qwen3-VL-32B-Instruct
Qwen3-VL-32B-Instruct fp8 需要48g显存
Qwen/Qwen3-VL-8B-Instruct
2. InternVideo2.5 ------ 目前非常成熟
InternVL2.5-26B-Instruct
30G显存
hf download OpenGVLab/InternVL2_5-26B-AWQ --local-dir ./OpenGVLab/InternVL2_5-26B-AWQ
yanziang/InternVideo3-8B-Instruct
python
import torch
from transformers import AutoModelForCausalLM, AutoProcessor
model_path = "OpenGVLab/InternVideo3-8B-Instruct"
model_path = "/data/feature/lbg/models/yanziang_InternVideo3-8B-Instruct/"
model = AutoModelForCausalLM.from_pretrained(
model_path,
dtype=torch.bfloat16,
attn_implementation="sdpa",
device_map="auto",
trust_remote_code=True,
)
processor = AutoProcessor.from_pretrained(
model_path,
trust_remote_code=True,
)
video_path = "/data/lbg/project/aigc/manim-generator/yumao_v/yq_highlight_1785620340669_124.mp4"
max_frames=128
min_frames=16
fps = 1
min_pixels = 128 * 32 * 32
max_pixels = 128 * 32 * 32
messages = [
{
"role": "user",
"content": [
{"type": "video", "video": video_path, "fps": fps},
{"type": "text", "text": "Please describe this video in detail."},
],
}
]
processor.video_processor.size = {
"longest_edge": max_pixels * max_frames,
"shortest_edge": min_pixels * min_frames,
}
inputs = processor.apply_chat_template(
messages,
tokenize=True,
add_generation_prompt=True,
return_dict=True,
fps=fps,
return_tensors="pt",
)
inputs = inputs.to(model.device)
output = model.generate(**inputs, max_new_tokens=1024, use_cache=True)
generated_ids = [o[len(i):] for i, o in zip(inputs.input_ids, output)]
print(processor.batch_decode(generated_ids, skip_special_tokens=True)[0])封装server：
python
# main.py
import torch
from transformers import AutoModelForCausalLM, AutoProcessor
from fastapi import FastAPI, HTTPException, UploadFile, File, Form
from fastapi.responses import JSONResponse
from pydantic import BaseModel
import tempfile
import os
import shutil
from typing import Optional, List
import base64
from io import BytesIO
from PIL import Image
import logging
from contextlib import asynccontextmanager
# 配置日志
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)
# 全局模型变量
model = None
processor = None
model_loaded = False
# 配置
MODEL_PATH = "/data/feature/lbg/models/yanziang_InternVideo3-8B-Instruct/"
MAX_FRAMES = 128
MIN_FRAMES = 16
MIN_PIXELS = 128 * 32 * 32
MAX_PIXELS = 128 * 32 * 32
MAX_NEW_TOKENS = 1024
# 请求模型
class VideoRequest(BaseModel):
video_path: str
fps: int = 1
max_new_tokens: int = MAX_NEW_TOKENS
prompt: str = "Please describe this video in detail."
max_frames: Optional[int] = MAX_FRAMES
min_frames: Optional[int] = MIN_FRAMES
class ImageRequest(BaseModel):
image_path: Optional[str] = None
prompt: str = "Please describe this image in detail."
max_new_tokens: int = MAX_NEW_TOKENS
class ChatRequest(BaseModel):
messages: List[dict]
max_new_tokens: int = MAX_NEW_TOKENS
fps: int = 1
# 响应模型
class InferenceResponse(BaseModel):
success: bool
result: Optional[str] = None
error: Optional[str] = None
def load_model():
"""加载模型和处理器"""
global model, processor, model_loaded
if model_loaded:
logger.info("Model already loaded")
return
try:
logger.info(f"Loading model from {MODEL_PATH}")
model = AutoModelForCausalLM.from_pretrained(
MODEL_PATH,
torch_dtype=torch.bfloat16,
attn_implementation="sdpa",
device_map="auto",
trust_remote_code=True,
)
processor = AutoProcessor.from_pretrained(
MODEL_PATH,
trust_remote_code=True,
)
model_loaded = True
logger.info("Model loaded successfully")
except Exception as e:
logger.error(f"Failed to load model: {str(e)}")
raise # 启动时加载失败，让应用启动失败
def unload_model():
"""卸载模型，释放资源"""
global model, processor, model_loaded
if model_loaded:
logger.info("Unloading model...")
model = None
processor = None
model_loaded = False
# 清理GPU缓存
if torch.cuda.is_available():
torch.cuda.empty_cache()
logger.info("Model unloaded")
# --- 使用 lifespan 替代 on_event ---
@asynccontextmanager
async def lifespan(app: FastAPI):
# 启动逻辑
logger.info("Application startup: Loading model...")
load_model()
yield
# 关闭逻辑
logger.info("Application shutdown: Cleaning up resources...")
unload_model()
# --- 创建 FastAPI 应用，传入 lifespan 参数 ---
app = FastAPI(
title="InternVideo3 API",
description="Video and Image Understanding API",
lifespan=lifespan
)
@app.get("/health")
async def health_check():
"""健康检查"""
return {"status": "healthy", "model_loaded": model_loaded}
@app.post("/predict/video_path", response_model=InferenceResponse)
async def predict_video_path(request: VideoRequest):
"""
使用视频路径进行推理
"""
if not model_loaded:
raise HTTPException(status_code=503, detail="Model not loaded")
try:
# 准备消息
messages = [
{
"role": "user",
"content": [
{"type": "video", "video": request.video_path, "fps": request.fps},
{"type": "text", "text": request.prompt},
],
}
]
# 设置视频处理器参数
processor.video_processor.size = {
"longest_edge": MAX_PIXELS * request.max_frames,
"shortest_edge": MIN_PIXELS * request.min_frames,
}
# 处理输入
inputs = processor.apply_chat_template(
messages,
tokenize=True,
add_generation_prompt=True,
return_dict=True,
fps=request.fps,
return_tensors="pt",
)
inputs = inputs.to(model.device)
# 生成输出
output = model.generate(**inputs, max_new_tokens=request.max_new_tokens, use_cache=True)
generated_ids = [o[len(i):] for i, o in zip(inputs.input_ids, output)]
result = processor.batch_decode(generated_ids, skip_special_tokens=True)[0]
return InferenceResponse(success=True, result=result)
except Exception as e:
logger.error(f"Prediction error: {str(e)}")
return InferenceResponse(success=False, error=str(e))
@app.post("/predict/video_upload", response_model=InferenceResponse)
async def predict_video_upload(
file: UploadFile = File(...),
prompt: str = Form("Please describe this video in detail."),
fps: int = Form(1),
max_new_tokens: int = Form(MAX_NEW_TOKENS)
):
"""
上传视频文件进行推理
"""
if not model_loaded:
raise HTTPException(status_code=503, detail="Model not loaded")
temp_file_path = None
try:
# 创建临时文件
with tempfile.NamedTemporaryFile(delete=False, suffix=".mp4") as temp_file:
shutil.copyfileobj(file.file, temp_file)
temp_file_path = temp_file.name
# 准备消息
messages = [
{
"role": "user",
"content": [
{"type": "video", "video": temp_file_path, "fps": fps},
{"type": "text", "text": prompt},
],
}
]
# 设置视频处理器参数
processor.video_processor.size = {
"longest_edge": MAX_PIXELS * MAX_FRAMES,
"shortest_edge": MIN_PIXELS * MIN_FRAMES,
}
# 处理输入
inputs = processor.apply_chat_template(
messages,
tokenize=True,
add_generation_prompt=True,
return_dict=True,
fps=fps,
return_tensors="pt",
)
inputs = inputs.to(model.device)
# 生成输出
output = model.generate(**inputs, max_new_tokens=max_new_tokens, use_cache=True)
generated_ids = [o[len(i):] for i, o in zip(inputs.input_ids, output)]
result = processor.batch_decode(generated_ids, skip_special_tokens=True)[0]
return InferenceResponse(success=True, result=result)
except Exception as e:
logger.error(f"Prediction error: {str(e)}")
return InferenceResponse(success=False, error=str(e))
finally:
# 清理临时文件
if temp_file_path and os.path.exists(temp_file_path):
os.unlink(temp_file_path)
if file.file:
file.file.close()
@app.post("/predict/image_path", response_model=InferenceResponse)
async def predict_image_path(request: ImageRequest):
"""
使用图像路径进行推理
"""
if not model_loaded:
raise HTTPException(status_code=503, detail="Model not loaded")
try:
# 准备消息
messages = [
{
"role": "user",
"content": [
{"type": "image", "image": request.image_path},
{"type": "text", "text": request.prompt},
],
}
]
# 处理输入
inputs = processor.apply_chat_template(
messages,
tokenize=True,
add_generation_prompt=True,
return_dict=True,
return_tensors="pt",
)
inputs = inputs.to(model.device)
# 生成输出
output = model.generate(**inputs, max_new_tokens=request.max_new_tokens, use_cache=True)
generated_ids = [o[len(i):] for i, o in zip(inputs.input_ids, output)]
result = processor.batch_decode(generated_ids, skip_special_tokens=True)[0]
return InferenceResponse(success=True, result=result)
except Exception as e:
logger.error(f"Prediction error: {str(e)}")
return InferenceResponse(success=False, error=str(e))
@app.post("/predict/image_upload", response_model=InferenceResponse)
async def predict_image_upload(
file: UploadFile = File(...),
prompt: str = Form("Please describe this image in detail."),
max_new_tokens: int = Form(MAX_NEW_TOKENS)
):
"""
上传图像文件进行推理
"""
if not model_loaded:
raise HTTPException(status_code=503, detail="Model not loaded")
temp_file_path = None
try:
# 创建临时文件
file_extension = os.path.splitext(file.filename)[1]
with tempfile.NamedTemporaryFile(delete=False, suffix=file_extension) as temp_file:
shutil.copyfileobj(file.file, temp_file)
temp_file_path = temp_file.name
# 准备消息
messages = [
{
"role": "user",
"content": [
{"type": "image", "image": temp_file_path},
{"type": "text", "text": prompt},
],
}
]
# 处理输入
inputs = processor.apply_chat_template(
messages,
tokenize=True,
add_generation_prompt=True,
return_dict=True,
return_tensors="pt",
)
inputs = inputs.to(model.device)
# 生成输出
output = model.generate(**inputs, max_new_tokens=max_new_tokens, use_cache=True)
generated_ids = [o[len(i):] for i, o in zip(inputs.input_ids, output)]
result = processor.batch_decode(generated_ids, skip_special_tokens=True)[0]
return InferenceResponse(success=True, result=result)
except Exception as e:
logger.error(f"Prediction error: {str(e)}")
return InferenceResponse(success=False, error=str(e))
finally:
# 清理临时文件
if temp_file_path and os.path.exists(temp_file_path):
os.unlink(temp_file_path)
if file.file:
file.file.close()
@app.post("/predict/chat", response_model=InferenceResponse)
async def predict_chat(request: ChatRequest):
"""
通用对话接口，支持多种输入类型
"""
if not model_loaded:
raise HTTPException(status_code=503, detail="Model not loaded")
try:
# 处理输入
inputs = processor.apply_chat_template(
request.messages,
tokenize=True,
add_generation_prompt=True,
return_dict=True,
fps=request.fps,
return_tensors="pt",
)
inputs = inputs.to(model.device)
# 生成输出
output = model.generate(**inputs, max_new_tokens=request.max_new_tokens, use_cache=True)
generated_ids = [o[len(i):] for i, o in zip(inputs.input_ids, output)]
result = processor.batch_decode(generated_ids, skip_special_tokens=True)[0]
return InferenceResponse(success=True, result=result)
except Exception as e:
logger.error(f"Chat prediction error: {str(e)}")
return InferenceResponse(success=False, error=str(e))
if __name__ == "__main__":
import uvicorn
uvicorn.run(
"qianwen_server:app",
host="0.0.0.0",
port=7999,
reload=False,
workers=1, # 由于模型占用大量内存，建议只使用1个worker
)