动作识别视频理解大模型羽毛球

原创已于 2026-08-15 17:01:27 修改 · 67 阅读

0 ·

本内容遵循CC 4.0 BY-SA版权协议

GEO检测

收录于

大模型

于 2026-08-14 21:51:29 首次发布

Qwen3-VL-32B-Instruct

Qwen/Qwen3-VL-8B-Instruct

2. InternVideo2.5 —— 目前非常成熟

InternVL2.5-26B-Instruct

yanziang/InternVideo3-8B-Instruct

羽毛动作识别封装server：

如果是在你前面这个**“视频动作识别 / 羽毛球视频分析”**场景里比较：

Qwen3-VL-32B-Instruct

Qwen3-VL-32B-Instruct fp8 需要48g显存

Qwen/Qwen3-VL-8B-Instruct

2. InternVideo2.5 —— 目前非常成熟

InternVL2.5-26B-Instruct

30G显存

hf download OpenGVLab/InternVL2_5-26B-AWQ --local-dir ./OpenGVLab/InternVL2_5-26B-AWQ

yanziang/InternVideo3-8B-Instruct 已封装

import torch
from transformers import AutoModelForCausalLM, AutoProcessor

model_path = "OpenGVLab/InternVideo3-8B-Instruct"
model_path = "/data/feature/lbg/models/yanziang_InternVideo3-8B-Instruct/"

model = AutoModelForCausalLM.from_pretrained(
    model_path,
    dtype=torch.bfloat16,
    attn_implementation="sdpa",
    device_map="auto",
    trust_remote_code=True,
)

processor = AutoProcessor.from_pretrained(
    model_path,
    trust_remote_code=True,
)

video_path = "/data/lbg/project/aigc/manim-generator/yumao_v/yq_highlight_1785620340669_124.mp4"

max_frames=128
min_frames=16
fps = 1
min_pixels = 128 * 32 * 32
max_pixels = 128 * 32 * 32

messages = [
    {
        "role": "user",
        "content": [
            {"type": "video", "video": video_path, "fps": fps},
            {"type": "text", "text": "Please describe this video in detail."},
        ],
    }
]

processor.video_processor.size = {
    "longest_edge": max_pixels * max_frames,
    "shortest_edge": min_pixels * min_frames,
}

inputs = processor.apply_chat_template(
    messages,
    tokenize=True,
    add_generation_prompt=True,
    return_dict=True,
    fps=fps,
    return_tensors="pt",
)
inputs = inputs.to(model.device)

output = model.generate(**inputs, max_new_tokens=1024, use_cache=True)
generated_ids = [o[len(i):] for i, o in zip(inputs.input_ids, output)]
print(processor.batch_decode(generated_ids, skip_special_tokens=True)[0])

羽毛动作识别封装server：

# main.py
import torch
from transformers import AutoModelForCausalLM, AutoProcessor
from fastapi import FastAPI, HTTPException, UploadFile, File, Form
from fastapi.responses import JSONResponse
from pydantic import BaseModel
import tempfile
import os
import shutil
from typing import Optional, List
import base64
from io import BytesIO
from PIL import Image
import logging
from contextlib import asynccontextmanager

# 配置日志
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)

# 全局模型变量
model = None
processor = None
model_loaded = False

# 配置
MODEL_PATH = "/data/feature/lbg/models/yanziang_InternVideo3-8B-Instruct/"
MAX_FRAMES = 128
MIN_FRAMES = 16
MIN_PIXELS = 512*512
MAX_PIXELS = 512*512
MAX_NEW_TOKENS = 1024

# 请求模型
class VideoRequest(BaseModel):
    video_path: str
    fps: int = 1
    max_new_tokens: int = MAX_NEW_TOKENS
    prompt: str = "Please describe this video in detail."
    max_frames: Optional[int] = MAX_FRAMES
    min_frames: Optional[int] = MIN_FRAMES

class ImageRequest(BaseModel):
    image_path: Optional[str] = None
    prompt: str = "Please describe this image in detail."
    max_new_tokens: int = MAX_NEW_TOKENS

class ChatRequest(BaseModel):
    messages: List[dict]
    max_new_tokens: int = MAX_NEW_TOKENS
    fps: int = 1

# 响应模型
class InferenceResponse(BaseModel):
    success: bool
    result: Optional[str] = None
    error: Optional[str] = None

def load_model():
    """加载模型和处理器"""
    global model, processor, model_loaded
    
    if model_loaded:
        logger.info("Model already loaded")
        return
    
    try:
        logger.info(f"Loading model from {MODEL_PATH}")
        
        model = AutoModelForCausalLM.from_pretrained(
            MODEL_PATH,
            torch_dtype=torch.bfloat16,
            attn_implementation="sdpa",
            device_map="auto",
            trust_remote_code=True,
        )
        
        processor = AutoProcessor.from_pretrained(
            MODEL_PATH,
            trust_remote_code=True,
        )
        
        model_loaded = True
        logger.info("Model loaded successfully")
        
    except Exception as e:
        logger.error(f"Failed to load model: {str(e)}")
        raise  # 启动时加载失败，让应用启动失败

def unload_model():
    """卸载模型，释放资源"""
    global model, processor, model_loaded
    if model_loaded:
        logger.info("Unloading model...")
        model = None
        processor = None
        model_loaded = False
        # 清理GPU缓存
        if torch.cuda.is_available():
            torch.cuda.empty_cache()
        logger.info("Model unloaded")

# --- 使用 lifespan 替代 on_event ---
@asynccontextmanager
async def lifespan(app: FastAPI):
    # 启动逻辑
    logger.info("Application startup: Loading model...")
    load_model()
    yield
    # 关闭逻辑
    logger.info("Application shutdown: Cleaning up resources...")
    unload_model()

# --- 创建 FastAPI 应用，传入 lifespan 参数 ---
app = FastAPI(
    title="InternVideo3 API",
    description="Video and Image Understanding API",
    lifespan=lifespan
)

@app.get("/health")
async def health_check():
    """健康检查"""
    return {"status": "healthy", "model_loaded": model_loaded}

@app.post("/predict/video_path", response_model=InferenceResponse)
async def predict_video_path(request: VideoRequest):

    if not model_loaded:
        raise HTTPException(status_code=503, detail="Model not loaded")
    
    try:
        # 准备消息
        messages = [
            {
                "role": "user",
                "content": [
                    {"type": "video", "video": request.video_path, "fps": request.fps},
                    {"type": "text", "text": request.prompt},
                ],
            }
        ]
        
        # 设置视频处理器参数
        processor.video_processor.size = {
            "longest_edge": MAX_PIXELS * request.max_frames,
            "shortest_edge": MIN_PIXELS * request.min_frames,
        }
        
        # 处理输入
        inputs = processor.apply_chat_template(
            messages,
            tokenize=True,
            add_generation_prompt=True,
            return_dict=True,
            fps=request.fps,
            return_tensors="pt",
        )
        inputs = inputs.to(model.device)
        
        # 生成输出
        output = model.generate(**inputs, max_new_tokens=request.max_new_tokens, use_cache=True)
        generated_ids = [o[len(i):] for i, o in zip(inputs.input_ids, output)]
        result = processor.batch_decode(generated_ids, skip_special_tokens=True)[0]
        
        return InferenceResponse(success=True, result=result)
        
    except Exception as e:
        logger.error(f"Prediction error: {str(e)}")
        return InferenceResponse(success=False, error=str(e))

@app.post("/predict/video_upload", response_model=InferenceResponse)
async def predict_video_upload(
    file: UploadFile = File(...),
    prompt: str = Form("Please describe this video in detail."),
    fps: int = Form(1),
    max_new_tokens: int = Form(MAX_NEW_TOKENS)
):
    """
    上传视频文件进行推理
    """
    if not model_loaded:
        raise HTTPException(status_code=503, detail="Model not loaded")
    
    temp_file_path = None
    
    try:
        # 创建临时文件
        with tempfile.NamedTemporaryFile(delete=False, suffix=".mp4") as temp_file:
            shutil.copyfileobj(file.file, temp_file)
            temp_file_path = temp_file.name
        
        # 准备消息
        messages = [
            {
                "role": "user",
                "content": [
                    {"type": "video", "video": temp_file_path, "fps": fps},
                    {"type": "text", "text": prompt},
                ],
            }
        ]
        
        # 设置视频处理器参数
        processor.video_processor.size = {
            "longest_edge": MAX_PIXELS * MAX_FRAMES,
            "shortest_edge": MIN_PIXELS * MIN_FRAMES,
        }
        
        # 处理输入
        inputs = processor.apply_chat_template(
            messages,
            tokenize=True,
            add_generation_prompt=True,
            return_dict=True,
            fps=fps,
            return_tensors="pt",
        )
        inputs = inputs.to(model.device)
        
        # 生成输出
        output = model.generate(**inputs, max_new_tokens=max_new_tokens, use_cache=True)
        generated_ids = [o[len(i):] for i, o in zip(inputs.input_ids, output)]
        result = processor.batch_decode(generated_ids, skip_special_tokens=True)[0]
        
        return InferenceResponse(success=True, result=result)
        
    except Exception as e:
        logger.error(f"Prediction error: {str(e)}")
        return InferenceResponse(success=False, error=str(e))
        
    finally:
        # 清理临时文件
        if temp_file_path and os.path.exists(temp_file_path):
            os.unlink(temp_file_path)
        if file.file:
            file.file.close()

@app.post("/predict/image_path", response_model=InferenceResponse)
async def predict_image_path(request: ImageRequest):
    """
    使用图像路径进行推理
    """
    if not model_loaded:
        raise HTTPException(status_code=503, detail="Model not loaded")
    
    try:
        # 准备消息
        messages = [
            {
                "role": "user",
                "content": [
                    {"type": "image", "image": request.image_path},
                    {"type": "text", "text": request.prompt},
                ],
            }
        ]
        
        # 处理输入
        inputs = processor.apply_chat_template(
            messages,
            tokenize=True,
            add_generation_prompt=True,
            return_dict=True,
            return_tensors="pt",
        )
        inputs = inputs.to(model.device)
        
        # 生成输出
        output = model.generate(**inputs, max_new_tokens=request.max_new_tokens, use_cache=True)
        generated_ids = [o[len(i):] for i, o in zip(inputs.input_ids, output)]
        result = processor.batch_decode(generated_ids, skip_special_tokens=True)[0]
        
        return InferenceResponse(success=True, result=result)
        
    except Exception as e:
        logger.error(f"Prediction error: {str(e)}")
        return InferenceResponse(success=False, error=str(e))

@app.post("/predict/image_upload", response_model=InferenceResponse)
async def predict_image_upload(
    file: UploadFile = File(...),
    prompt: str = Form("Please describe this image in detail."),
    max_new_tokens: int = Form(MAX_NEW_TOKENS)
):
    """
    上传图像文件进行推理
    """
    if not model_loaded:
        raise HTTPException(status_code=503, detail="Model not loaded")
    
    temp_file_path = None
    
    try:
        # 创建临时文件
        file_extension = os.path.splitext(file.filename)[1]
        with tempfile.NamedTemporaryFile(delete=False, suffix=file_extension) as temp_file:
            shutil.copyfileobj(file.file, temp_file)
            temp_file_path = temp_file.name
        
        # 准备消息
        messages = [
            {
                "role": "user",
                "content": [
                    {"type": "image", "image": temp_file_path},
                    {"type": "text", "text": prompt},
                ],
            }
        ]
        
        # 处理输入
        inputs = processor.apply_chat_template(
            messages,
            tokenize=True,
            add_generation_prompt=True,
            return_dict=True,
            return_tensors="pt",
        )
        inputs = inputs.to(model.device)
        
        # 生成输出
        output = model.generate(**inputs, max_new_tokens=max_new_tokens, use_cache=True)
        generated_ids = [o[len(i):] for i, o in zip(inputs.input_ids, output)]
        result = processor.batch_decode(generated_ids, skip_special_tokens=True)[0]
        
        return InferenceResponse(success=True, result=result)
        
    except Exception as e:
        logger.error(f"Prediction error: {str(e)}")
        return InferenceResponse(success=False, error=str(e))
        
    finally:
        # 清理临时文件
        if temp_file_path and os.path.exists(temp_file_path):
            os.unlink(temp_file_path)
        if file.file:
            file.file.close()

@app.post("/predict/chat", response_model=InferenceResponse)
async def predict_chat(request: ChatRequest):
    """
    通用对话接口，支持多种输入类型
    """
    if not model_loaded:
        raise HTTPException(status_code=503, detail="Model not loaded")
    
    try:
        # 处理输入
        inputs = processor.apply_chat_template(
            request.messages,
            tokenize=True,
            add_generation_prompt=True,
            return_dict=True,
            fps=request.fps,
            return_tensors="pt",
        )
        inputs = inputs.to(model.device)
        
        # 生成输出
        output = model.generate(**inputs, max_new_tokens=request.max_new_tokens, use_cache=True)
        generated_ids = [o[len(i):] for i, o in zip(inputs.input_ids, output)]
        result = processor.batch_decode(generated_ids, skip_special_tokens=True)[0]
        
        return InferenceResponse(success=True, result=result)
        
    except Exception as e:
        logger.error(f"Chat prediction error: {str(e)}")
        return InferenceResponse(success=False, error=str(e))

if __name__ == "__main__":
    import uvicorn
    uvicorn.run(
        "qianwen_server:app",
        host="0.0.0.0",
        port=7999,
        reload=False,
        workers=1,  # 由于模型占用大量内存，建议只使用1个worker
    )

客户端判断捡球：

import glob
import os
import shutil

# client_example.py
import requests
import json

# 基础URL
BASE_URL = "http://localhost:8000"
BASE_URL = "http://39.97.230.23:7999"

# 1. 使用视频路径
def predict_video_path(vidio_path,prompt):
    url = f"{BASE_URL}/predict/video_path"
    data = {
        "video_path": vidio_path,
        "fps": 1,
        "max_new_tokens": 1024,
        "prompt": prompt
    }
    response = requests.post(url, json=data)

    try:
        return response.json()
    except json.JSONDecodeError as e:
        print(f"JSON解析失败: {e}")
        print(f"完整响应内容: {response.text}")
        return None

# 2. 上传视频文件
def predict_video_upload(vidio_path,prompt):
    url = f"{BASE_URL}/predict/video_upload"
    files = {
        'file': ('video.mp4', open(vidio_path, 'rb'), 'video/mp4')
    }
    data = {
        'prompt': prompt,
        'fps': 1,
        'max_new_tokens': 1024
    }
    response = requests.post(url, files=files, data=data)

    print(response.json())


# 4. 上传图像文件
def predict_image_upload():
    url = f"{BASE_URL}/predict/image_upload"
    files = {
        'file': ('image.jpg', open(r"C:\Users\ChanJing-01\Pictures\jihe\6f2ce46f-b1a9-4f76-b62d-396ca15f8160.png", 'rb'), 'image/jpeg')
    }
    data = {
        # 'prompt': 'What is in this image?',
        # 'prompt': "请描述这个图像中物体的位置，使用相对位置描述（如：左上角、中心、右下角等）。请用中文回答。",
        # 'prompt': "请找出图像中的几何图，并以 [x_min, y_min, x_max, y_max] 的格式输出其边界框坐标。坐标范围是0-1000。如果图像中有多个物体，请列出所有物体及其坐标。",
        'prompt': "请获取图像的大小分辨率",
        'max_new_tokens': 512
    }
    response = requests.post(url, files=files, data=data)
    print(response.json())


# 5. 健康检查
def health_check():
    response = requests.get(f"{BASE_URL}/health")
    print(response.json())


def  video_server():
   serer_base=r"/data/feature/lbg/data/yuqiu//"
   serer_base=r"/data/feature/lbg/data//segments_2/"

   mp4_files = glob.glob(fr"C:\Users\ChanJing-01\Videos\yumao\yuqiu/*.mp4")
   mp4_files = glob.glob(fr"E:\project\track\pytracking-master\segments_2/*.mp4")

   prompt = """判断视频中是否显示"从地面捡起羽毛球"的动作。
           捡球 = 球员从地面捡起一个**静止**的羽毛球，动作**平稳缓慢**。
           - 球员主动弯腰，用手或球拍拿起球，手臂倾斜向下指向地面
           - 动作结束后不再举起手臂击球
           不算捡球的情况：
           - 球在飞行中，球员去接球/救球（无论身体是否倾斜）
           - 任何有击球挥拍的动作
           - 快速扑出或快速倾斜
           - 身体倾斜去击球，手臂水平或者向上
           关键区分：看球是静止还是运动。球在动 = 不是捡球。
           回答：
           结果：是 或 否
           描述：说明球的状态（静止/运动）和球员动作"""

   for mp4_file in mp4_files:

       file_name=os.path.basename(mp4_file)
       file_path=serer_base+'/'+file_name
       res_json =  predict_video_path(file_path, prompt)

       if res_json is not None:
           action = res_json.get('result',"")

           dir_ok="jianqiu"
           dir_no='jiqiu'

           os.makedirs(dir_ok,exist_ok=True)
           os.makedirs(dir_no,exist_ok=True)
           if "是" in action:
               shutil.copy(mp4_file,dir_ok+'/'+file_name)
           else:
               shutil.copy(mp4_file,dir_no+'/'+file_name)

           print(action, mp4_file)
       else:
           print('none',mp4_file)


if __name__ == "__main__":
    # 健康检查
    health_check()
    video_server()
    predict_video_upload(r"E:\pro_math\math_image\qianwenvl\jianqiu_no\wenzi_crop_013.mp4","不打球一般会朝喜爱，移动速度较快，视频中的人手臂朝上吗？是要准备击球吗")

标签

#音视频 #python #开发语言