【大模型】-LangChain多模态输入和自定义输出

文章目录

- - - [11.1 xml输出](#11.1 xml输出)
    - [11.2 json输出](#11.2 json输出)
    - [11.3 支持图片输入](#11.3 支持图片输入)
    - [11.4 配合工具输出](#11.4 配合工具输出)

11.1 xml输出

python 复制代码

from langchain_core.output_parsers import JsonOutputParser, XMLOutputParser
from langchain_core.prompts import ChatPromptTemplate, HumanMessagePromptTemplate, MessagesPlaceholder, PromptTemplate
from langchain_community.chat_models import ChatTongyi

from dotenv import load_dotenv
from pydantic import BaseModel, Field

load_dotenv()

llm = ChatTongyi(model="qwen-plus")


joke_query = "生成周杰伦的歌曲，按照年份升序排列"
parser =XMLOutputParser()
prompt = PromptTemplate(
    template="回答用户的查询。\n{format_instructions}\n{query}\n",
    input_variables=["query"],
    partial_variables={"format_instructions": parser.get_format_instructions()},
)

chain = prompt | llm
response = chain.invoke({"query": joke_query})
print(response.content)

#输出结果
<songs>
  <song>
    <title>可爱女人</title>
    <year>2000</year>
    <album>Jay</album>
  </song>
  <song>
    <title>星晴</title>
    <year>2000</year>
    <album>Jay</album>
  </song>
  <song>
    <title>龙卷风</title>
    <year>2000</year>
    <album>Jay</album>
  </song>
  <song>
    <title>简单爱</title>
    <year>2001</year>
    <album>Fantasy</album>
  </song>
  <song>
    <title>开不了口</title>
    <year>2001</year>
    <album>Fantasy</album>
  </song>
  <song>
    <title>双截棍</title>
    <year>2001</year>
    <album>Fantasy</album>
  </song>
  <song>
    <title>爱在西元前</title>
    <year>2001</year>
    <album>Fantasy</album>
  </song>
  <song>
    <title>娘子</title>
    <year>2000</year>
    <album>Jay</album>
  </song>
  <song>
    <title>安静</title>
    <year>2000</year>
    <album>Jay</album>
  </song>
  <song>
    <title>半岛铁盒</title>
    <year>2002</year>
    <album>The Eight Dimensions</album>
  </song>
  <song>
    <title>东风破</title>
    <year>2003</year>
    <album>Yeh Hui-Mei</album>
  </song>
  <song>
    <title>晴天</title>
    <year>2003</year>
    <album>Yeh Hui-Mei</album>
  </song>
  <song>
    <title>七里香</title>
    <year>2004</year>
    <album>Common Jasmine Orange</album>
  </song>
  <song>
    <title>发如雪</title>
    <year>2005</year>
    <album>November's Chopin</album>
  </song>
  <song>
    <title>千里之外</title>
    <year>2006</year>
    <album>Still Fantasy</album>
  </song>
  <song>
    <title>青花瓷</title>
    <year>2007</year>
    <album>On the Run!</album>
  </song>
  <song>
    <title>稻香</title>
    <year>2008</year>
    <album>Capricorn</album>
  </song>
  <song>
    <title>说好不哭</title>
    <year>2019</year>
    <album>未发行专辑</album>
  </song>
  <song>
    <title>Mojito</title>
    <year>2020</year>
    <album>Mojito</album>
  </song>
  <song>
    <title>最伟大的作品</title>
    <year>2022</year>
    <album>最伟大的作品</album>
  </song>
</songs>

11.2 json输出

python 复制代码

from langchain_core.output_parsers import JsonOutputParser
from langchain_core.prompts import ChatPromptTemplate, HumanMessagePromptTemplate, MessagesPlaceholder, PromptTemplate
from langchain_community.chat_models import ChatTongyi

from dotenv import load_dotenv
from pydantic import BaseModel, Field

load_dotenv()

llm = ChatTongyi(model="qwen-plus")

class Joke(BaseModel):
    setup: str = Field(description="设置笑话的问题")
    punchline: str = Field(description="解决笑话的答案")

joke_query = "告诉我一个笑话"
parser =JsonOutputParser(pydantic_object=Joke)
prompt = PromptTemplate(
    template="回答用户的查询。\n{format_instructions}\n{query}\n",
    input_variables=["query"],
    partial_variables={"format_instructions": parser.get_format_instructions()},
)

chain = prompt | llm | parser
##print(chain.invoke({"query": joke_query}))
for chunk in chain.stream({"query": joke_query}):
    print(chunk)
    
#输出
{}
{'setup': ''}
{'setup': '为什么'}
{'setup': '为什么程序员总是喜欢用黑暗'}
{'setup': '为什么程序员总是喜欢用黑暗模式？'}
{'setup': '为什么程序员总是喜欢用黑暗模式？', 'punchline': '因为他们'}
{'setup': '为什么程序员总是喜欢用黑暗模式？', 'punchline': '因为他们不喜欢光明。'}

11.3 支持图片输入

python 复制代码

import base64
import httpx
from langchain_core.messages import HumanMessage
from langchain_community.chat_models import ChatTongyi
from dotenv import load_dotenv
load_dotenv()


image_url = "https://img1.baidu.com/it/u=352739982,3234821554&fm=253&app=138&f=JPEG?w=500&h=857"
image_data = base64.b64encode(httpx.get(image_url).content).decode("utf-8")
llm = ChatTongyi(model="qwen-plus")  # 使用支持图像识别的模型

message = HumanMessage(
    content=[
        {"type": "text", "text": "用中文描述这张图片的内容"},
        {
            "type": "image_url",
            "image_url": {
                "url": {"url": f"data:image/jpeg;base64,{image_data}"}
            }
        }
    ]
)

response = llm.invoke([message])
print(response.content)


#输出
这张图片展示了一个现代风格的客厅，整体色调以白色和浅木色为主，营造出明亮、宽敞的空间感。以下是图片中各个元素的详细描述：

1. **地板**：  
   地板采用浅色木地板，表面光滑且有自然的木纹纹理，增加了空间的温馨感。

2. **沙发**：  
   客厅中央摆放着一组灰色布艺沙发，包括一个三人座沙发和两个单人座沙发。沙发的设计简洁，线条流畅，坐垫柔软，给人一种舒适的感觉。沙发上没有多余的装饰，保持了极简主义的风格。

3. **茶几**：  
   沙发前有一张圆形玻璃茶几，透明的桌面让空间显得更加通透。茶几上放置了一本打开的书和一个小型植物盆栽，增添了一丝生活气息。
    
...............

11.4 配合工具输出

python 复制代码

import base64
import httpx
from langchain_core.messages import HumanMessage, ToolMessage
from langchain_community.chat_models import ChatTongyi
from langchain_core.tools import tool  # 补充导入tool装饰器
from typing import Literal  # 添加此行以解决 Literal 未定义问题
from dotenv import load_dotenv
load_dotenv()


image_url = "https://copyright.bdstatic.com/vcg/creative/cae2f3d12fc98f2eb94d625dad4f42a3.jpg@wm_1,k_cGljX2JqaHdhdGVyLmpwZw=="
image_data = base64.b64encode(httpx.get(image_url).content).decode("utf-8")
llm = ChatTongyi(model="qwen-plus")  # 使用支持图像识别的模型

@tool
def weather_tool(weather: Literal["晴朗的", "多云的", "多雨的", "下雪的"]) -> str:
    """根据天气类型返回描述"""
    weather_descriptions = {
        "晴朗的": "今天阳光明媚，是个好天气。",
        "多云的": "今天多云，可能会有些阴沉。",
        "多雨的": "今天有雨，记得带伞。",
        "下雪的": "今天下雪，注意保暖和防滑。"
    }
    return weather_descriptions.get(weather, "无法识别的天气类型。")

model_with_tools = llm.bind_tools(tools=[weather_tool])

message = HumanMessage(
    content=[
        {"type": "text", "text": "用中文描述这张图片的天气"},
        {
            "type": "image_url",
            "image_url": f"data:image/jpeg;base64,{image_data}"  # 直接使用字符串
        }
    ]
)

response = model_with_tools.invoke([message])
print(response.content)

# 创建包含工具调用的完整链路
# 修改工具调用处理部分
# 修改工具调用处理部分，确保消息格式正确
# def call_model_with_tools(messages):
#     response = model_with_tools.invoke(messages)
#
#     # 检查是否有工具调用
#     if hasattr(response, 'tool_calls') and response.tool_calls:
#         tool_results = []
#         for tool_call in response.tool_calls:
#             # 访问字典中的参数（如果tool_call是字典）
#             if isinstance(tool_call, dict):
#                 tool_name = tool_call.get('name')
#                 tool_args = tool_call.get('args')
#                 tool_call_id = tool_call.get('id')
#             else:
#                 # 如果tool_call是对象
#                 tool_name = getattr(tool_call, 'name', None)
#                 tool_args = getattr(tool_call, 'args', {})
#                 tool_call_id = getattr(tool_call, 'id', None)
#
#             # 根据工具名称执行相应工具
#             if tool_name == 'weather_tool':
#                 try:
#                     result = weather_tool.invoke(tool_args)
#                     tool_results.append(
#                         ToolMessage(
#                             content=str(result),
#                             tool_call_id=tool_call_id
#                         )
#                     )
#                 except Exception as e:
#                     tool_results.append(
#                         ToolMessage(
#                             content=f"工具执行错误: {str(e)}",
#                             tool_call_id=tool_call_id
#                         )
#                     )
#
#         # 将工具结果添加到消息历史中，并重新调用模型
#         new_messages = messages + [response] + tool_results
#         return call_model_with_tools(new_messages)
#
#     return response
#
# # 使用完整工具调用链路
# final_response = call_model_with_tools([message])
# print(final_response.content)