Electron Forge【实战】带图片的 AI 聊天

改用支持图片的 AI 模型

qwen-turbo 仅支持文字，要想体验图片聊天，需改用 qwen-vl-plus

src/initData.ts

  {
    id: 2,
    name: "aliyun",
    title: "阿里 -- 通义千问",
    desc: "阿里百炼 -- 通义千问",
    // https://help.aliyun.com/zh/dashscope/developer-reference/api-details?spm=a2c4g.11186623.0.0.5bf41507xgULX5#b148acc634pfc
    models: ["qwen-turbo", "qwen-vl-plus"],
    avatar:
      "https://qph.cf2.poecdn.net/main-thumb-pb-4160791-200-qlqunomdvkyitpedtghnhsgjlutapgfl.jpeg",
  },

安装依赖 mime-types

用于便捷获取图片的类型

npm i mime-types @types/mime-types --save-dev

提问框中选择本地图片

src/components/MessageInput.vue

<template>
  <div
    class="message-input w-full shadow-sm border rounded-lg border-gray-300 py-1 px-2 focus-within:border-green-700"
  >
    <div v-if="imagePreview" class="my-2 relative inline-block">
      <img
        :src="imagePreview"
        alt="Preview"
        class="h-24 w-24 object-cover rounded"
      />
      <Icon
        icon="lets-icons:dell-fill"
        width="24"
        @click="delImg"
        class="absolute top-[-10px] right-[-10px] p-1 rounded-full cursor-pointer"
      />
    </div>
    <div class="flex items-center">
      <input
        type="file"
        accept="image/*"
        ref="fileInput"
        class="hidden"
        @change="handleImageUpload"
      />
      <Icon
        icon="radix-icons:image"
        width="24"
        height="24"
        :class="[
          'mr-2',
          disabled
            ? 'text-gray-300 cursor-not-allowed'
            : 'text-gray-400 cursor-pointer hover:text-gray-600',
        ]"
        @click="triggerFileInput"
      />
      <input
        class="outline-none border-0 flex-1 bg-white focus:ring-0"
        type="text"
        ref="ref_input"
        v-model="model"
        :disabled="disabled"
        :placeholder="tip"
        @keydown.enter="onCreate"
      />
      <Button
        icon-name="radix-icons:paper-plane"
        @click="onCreate"
        :disabled="disabled"
      >
        发送
      </Button>
    </div>
  </div>
</template>

<script lang="ts" setup>
import { ref } from "vue";
import { Icon } from "@iconify/vue";

import Button from "./Button.vue";

const props = defineProps<{
  disabled?: boolean;
}>();
const emit = defineEmits<{
  create: [value: string, imagePath?: string];
}>();
const model = defineModel<string>();
const fileInput = ref<HTMLInputElement | null>(null);
const imagePreview = ref("");
const triggerFileInput = () => {
  if (!props.disabled) {
    fileInput.value?.click();
  }
};
const tip = ref("");
let selectedImage: File | null = null;
const handleImageUpload = (event: Event) => {
  const target = event.target as HTMLInputElement;
  if (target.files && target.files.length > 0) {
    selectedImage = target.files[0];
    const reader = new FileReader();
    reader.onload = (e) => {
      imagePreview.value = e.target?.result as string;
    };
    reader.readAsDataURL(selectedImage);
  }
};
const onCreate = async () => {
  if (model.value && model.value.trim() !== "") {
    if (selectedImage) {
      const filePath = window.electronAPI.getFilePath(selectedImage);
      emit("create", model.value, filePath);
    } else {
      emit("create", model.value);
    }

    selectedImage = null;
    imagePreview.value = "";
  } else {
    tip.value = "请输入问题";
  }
};
const ref_input = ref<HTMLInputElement | null>(null);

const delImg = () => {
  selectedImage = null;
  imagePreview.value = "";
};

defineExpose({
  ref_input: ref_input,
});
</script>

<style scoped>
input::placeholder {
  color: red;
}
</style>

src/preload.ts

需借助 webUtils 从 File 对象中获取文件路径

import { ipcRenderer, contextBridge, webUtils } from "electron";

 getFilePath: (file: File) => webUtils.getPathForFile(file),

将选择的图片，转存到应用的用户目录

图片很占空间，转为字符串直接存入数据库压力过大，合理的方案是存到应用本地

src/views/Home.vue

在创建会话时执行

const createConversation = async (question: string, imagePath?: string) => {
  const [AI_providerName, AI_modelName] = currentProvider.value.split("/");

  let copiedImagePath: string | undefined;
  if (imagePath) {
    try {
      copiedImagePath = await window.electronAPI.copyImageToUserDir(imagePath);
    } catch (error) {
      console.error("拷贝图片失败:", error);
    }
  }

  // 用 dayjs 得到格式化的当前时间字符串
  const currentTime = dayjs().format("YYYY-MM-DD HH:mm:ss");

  // pinia 中新建会话，得到新的会话id
  const conversationId = await conversationStore.createConversation({
    title: question,
    AI_providerName,
    AI_modelName,
    createdAt: currentTime,
    updatedAt: currentTime,
    msgList: [
      {
        type: "question",
        content: question,
        // 如果有图片路径，则将其添加到消息中
        ...(copiedImagePath && { imagePath: copiedImagePath }),
        createdAt: currentTime,
        updatedAt: currentTime,
      },
      {
        type: "answer",
        content: "",
        status: "loading",
        createdAt: currentTime,
        updatedAt: currentTime,
      },
    ],
  });

  // 更新当前选中的会话
  conversationStore.selectedId = conversationId;

  // 右侧界面--跳转到会话页面 -- 带参数 init 为新创建的会话的第一条消息id
  router.push(`/conversation/${conversationId}?type=new`);
};

src/preload.ts

  // 拷贝图片到本地用户目录
  copyImageToUserDir: (sourcePath: string) =>
    ipcRenderer.invoke("copy-image-to-user-dir", sourcePath),

src/ipc.ts

  // 拷贝图片到本地用户目录
  ipcMain.handle(
    "copy-image-to-user-dir",
    async (event, sourcePath: string) => {
      const userDataPath = app.getPath("userData");
      const imagesDir = path.join(userDataPath, "images");
      await fs.mkdir(imagesDir, { recursive: true });
      const fileName = path.basename(sourcePath);
      const destPath = path.join(imagesDir, fileName);
      await fs.copyFile(sourcePath, destPath);
      return destPath;
    }
  );

将图片信息传给 AI

src/views/Conversation.vue

发起 AI 聊天传图片参数

// 访问 AI 模型，获取答案
const get_AI_answer = async (answerIndex: number) => {
  await window.electronAPI.startChat({
    messageId: answerIndex,
    providerName: convsersation.value!.AI_providerName,
    selectedModel: convsersation.value!.AI_modelName,
    // 发给AI模型的消息需移除最后一条加载状态的消息,使最后一条消息为用户的提问
    messages: convsersation
      .value!.msgList.map((message) => ({
        role: message.type === "question" ? "user" : "assistant",
        content: message.content,
        // 若有图片信息，则将其添加到消息中
        ...(message.imagePath && { imagePath: message.imagePath }),
      }))
      .slice(0, -1),
  });
};

继续向 AI 提问时图片参数

const sendNewMessage = async (question: string, imagePath?: string) => {
  let copiedImagePath: string | undefined;
  if (imagePath) {
    try {
      copiedImagePath = await window.electronAPI.copyImageToUserDir(imagePath);
    } catch (error) {
      console.error("拷贝图片失败:", error);
    }
  }

  // 获取格式化的当前时间
  let currentTime = dayjs().format("YYYY-MM-DD HH:mm:ss");

  // 向消息列表中追加新的问题
  convsersation.value!.msgList.push({
    type: "question",
    content: question,
    ...(copiedImagePath && { imagePath: copiedImagePath }),
    createdAt: currentTime,
    updatedAt: currentTime,
  });

  // 向消息列表中追加 loading 状态的回答
  let new_msgList_length = convsersation.value!.msgList.push({
    type: "answer",
    content: "",
    createdAt: currentTime,
    updatedAt: currentTime,
    status: "loading",
  });

  // 消息列表的最后一条消息为 loading 状态的回答，其id为消息列表的长度 - 1
  let loading_msg_id = new_msgList_length - 1;

  // 访问 AI 模型获取答案，参数为 loading 状态的消息的id
  get_AI_answer(loading_msg_id);

  // 清空问题输入框
  inputValue.value = "";

  await messageScrollToBottom();

  // 发送问题后，问题输入框自动聚焦
  if (dom_MessageInput.value) {
    dom_MessageInput.value.ref_input.focus();
  }
};

src/providers/OpenAIProvider.ts

将消息转换为 AI 模型需要的格式后传给 AI

import OpenAI from "openai";
import { convertMessages } from "../util";

interface ChatMessageProps {
  role: string;
  content: string;
  imagePath?: string;
}

interface UniversalChunkProps {
  is_end: boolean;
  result: string;
}

export class OpenAIProvider {
  private client: OpenAI;
  constructor(apiKey: string, baseURL: string) {
    this.client = new OpenAI({
      apiKey,
      baseURL,
    });
  }
  async chat(messages: ChatMessageProps[], model: string) {
    // 将消息转换为AI模型需要的格式
    const convertedMessages = await convertMessages(messages);
    const stream = await this.client.chat.completions.create({
      model,
      messages: convertedMessages as any,
      stream: true,
    });
    const self = this;
    return {
      async *[Symbol.asyncIterator]() {
        for await (const chunk of stream) {
          yield self.transformResponse(chunk);
        }
      },
    };
  }
  protected transformResponse(
    chunk: OpenAI.Chat.Completions.ChatCompletionChunk
  ): UniversalChunkProps {
    const choice = chunk.choices[0];
    return {
      is_end: choice.finish_reason === "stop",
      result: choice.delta.content || "",
    };
  }
}

src/util.ts

函数封装 -- 将消息转换为 AI 模型需要的格式

import fs from 'fs/promises'
import { lookup } from 'mime-types'
export async function convertMessages( messages:  { role: string; content: string, imagePath?: string}[]) {
  const convertedMessages = []
  for (const message of messages) {
    let convertedContent: string | any[]
    if (message.imagePath) {
      const imageBuffer = await fs.readFile(message.imagePath)
      const base64Image = imageBuffer.toString('base64')
      const mimeType = lookup(message.imagePath)
      convertedContent = [
        {
          type: "text",
          text: message.content || ""
        },
        {
          type: 'image_url',
          image_url: {
            url: `data:${mimeType};base64,${base64Image}`
          }
        }
      ]
    } else {
      convertedContent = message.content
    }
    const { imagePath, ...messageWithoutImagePath } = message
    convertedMessages.push({
      ...messageWithoutImagePath,
      content: convertedContent
    })
  }
  return convertedMessages
}

加载消息记录中的图片

渲染进程中，无法直接读取本地图片，需借助 protocol 实现

src/main.ts

import { app, BrowserWindow, protocol, net } from "electron";
import { pathToFileURL } from "node:url";
import path from "node:path";

// windows 操作系统必要
protocol.registerSchemesAsPrivileged([
  {
    scheme: "safe-file",
    privileges: {
      standard: true,
      secure: true,
      supportFetchAPI: true,
    },
  },
]);

在 createWindow 方法内执行

  protocol.handle("safe-file", async (request) => {
    const userDataPath = app.getPath("userData");
    const imageDir = path.join(userDataPath, "images");
    // 去除协议头 safe-file://，解码 URL 中的路径
    const filePath = path.join(
      decodeURIComponent(request.url.slice("safe-file:/".length))
    );
    const filename = path.basename(filePath);
    const fileAddr = path.join(imageDir, filename);
    // 转换为 file:// URL
    const newFilePath = pathToFileURL(fileAddr).toString();
    // 使用 net.fetch 加载本地文件
    return net.fetch(newFilePath);
  });

页面中渲染图片

src/components/MessageList.vue

img 的 src 添加了 safe-file:// 协议

          <div v-if="message.type === 'question'">
            <div class="mb-3 flex justify-end">
              <img
                v-if="message.imagePath"
                :src="`safe-file://${message.imagePath}`"
                alt="提问的配图"
                class="h-24 w-24 object-cover rounded"
              />
            </div>
            <div
              class="message-question bg-green-700 text-white p-2 rounded-md"
            >
              {{ message.content }}
            </div>
          </div>

最终效果