Api For LLM

py 复制代码

# 构建虚拟环境 
conda create --name testapi python==3.10 
# 进入虚拟环境
conda activate testapi
# 更新一下
python -m pip install --upgrade pip
# 安装库
pip3 install "fschat[model_worker,webui]"
# 输入以下命令
# 启动控制器服务
python3 -m fastchat.serve.controller --host 127.0.0.1
# 启动模型服务
python3 -m fastchat.serve.model_worker --model-path ./Llama-2-70b-chat-hf --num-gpus 7 --host 127.0.0.1  --worker-address http://127.0.0.1:21002 --controller-address http://127.0.0.1:21001  # 请注意输出中的模型名称，用于调用
# 启动API服务
python3 -m fastchat.serve.openai_api_server --host 127.0.0.1 --port 8000
# 启动web服务（未尝试是否可以）服务默认端口是 7860，可以通过--port参数来修改端口，还可以通过添加--share参数来开启 Gradio 的共享模式，这样就可以通过外网访问 WebUI 服务了
python -m fastchat.serve.gradio_web_server --host 0.0.0.0

python 复制代码

# 使用openai 进行调用
# fastchat 官方文档：https://github.com/lm-sys/FastChat/blob/main/docs/openai_api.md
import openai

openai.api_key = "EMPTY"
openai.base_url = "http://localhost:8000/v1/"

model = "Llama-2-70b-chat-hf"   
prompt = "Once upon a time"

# create a completion
completion = openai.completions.create(model=model, prompt=prompt, max_tokens=64)
# print the completion
print(prompt + completion.choices[0].text)

# create a chat completion
completion = openai.chat.completions.create(
  model=model,
  messages=[{"role": "user", "content": "Hello! What is your name?"}]
)
# print the completion
print(completion.choices[0].message.content)


## 如果输出你觉得没有完成，请再次访问
# completion = openai.chat.completions.create(
#   model=model,
#   messages=[{"role": "user", "content": "Hello! What is your name?"},
#             {"role": "assistant", "content": completion.choices[0].message.content},
#            {"role": "user", "content": "Continue."}]
# )
# # print the completion
# print(completion.choices[0].message.content)

py 复制代码

pip install uvicorn
pip install fastapi 
pip install pydantic 
pip install torch
pip install transformers

py 复制代码

# 服务端使用文件
import uvicorn
from fastapi import FastAPI
from pydantic import BaseModel
import uvicorn, json, datetime
import torch
import os
from transformers import AutoModelForCausalLM, AutoTokenizer
from transformers.generation.utils import GenerationConfig
os.environ['CUDA_VISIBLE_DEVICES'] = "5,6,7"

app = FastAPI()

class Query(BaseModel):
    text: str

path = "/workdir/model/baichuan13b_chat/"
tokenizer = AutoTokenizer.from_pretrained(path, use_fast=False, trust_remote_code=True)
model = AutoModelForCausalLM.from_pretrained(path, device_map="auto", torch_dtype=torch.float16, trust_remote_code=True)
model.generation_config = GenerationConfig.from_pretrained(path)


@app.post("/chat/")
async def chat(query: Query):
    input_ids = tokenizer([query.text]).input_ids
    output_ids = model.generate(
        torch.as_tensor(input_ids).cuda(),
        do_sample=False,
        temperature=0.1,
        repetition_penalty=1,
        max_new_tokens=1024)
    output_ids = output_ids[0][len(input_ids[0]):]
    outputs = tokenizer.decode(output_ids, skip_special_tokens=True, spaces_between_special_tokens=False)
    return {"result": outputs}


if __name__ == "__main__":
    uvicorn.run(app, host="0.0.0.0", port=6667)

python 复制代码

# 访问调用
import requests

url = "http://0.0.0.0:6667/chat/"
query = {"text": "你好，请做一段自我介绍。"}

response = requests.post(url, json=query)

if response.status_code == 200:
    result = response.json()
    print("BOT:", result["result"])
else:
    print("Error:", response.status_code, response.text)

另一种方法