12.6深度学习_模型优化和迁移_整体流程梳理

七、整体流程梳理

1. 引入使用的包

用到什么包,临时引入就可以,不用太担心。

python 复制代码
import time
import os

import numpy as np
import pandas as pd
import torch
import torch.nn as nn
import torch.optim as optim
import torchvision
import torchvision.transforms as transforms
from torch.utils.data import DataLoader
from torchvision.datasets import CIFAR10

from torchvision.models import resnet18, ResNet18_Weights
import wandb
from torch.utils.tensorboard import SummaryWriter
from sklearn.metrics import *
import matplotlib.pyplot as plt

2. 数据

python 复制代码
# 下面和以前就一样了
train_dataset = CIFAR10(
    root=datapath,
    train=True,
    download=True,
    transform=transform,
)
# 构建训练数据集
train_loader = DataLoader(
    #
    dataset=train_dataset,
    batch_size=batzh_size,
    shuffle=True,
    num_workers=2,
)

3. 模型

python 复制代码
# 再次获取resnet18原始神经网络并对齐fc层进行调整
model = resnet18(weights=None)

in_features = model.fc.in_features
# 重写FC:我们这里做的是10分类
model.fc = nn.Linear(in_features=in_features, out_features=10)

# 需要对权重信息进行处理:要加载我们训练之后最新的权重文件
weights_default = torch.load(weightpath)
weights_default.pop("fc.weight")
weights_default.pop("fc.bias")

# 把权重参数进行同步
new_state_dict = model.state_dict()
weights_default_process = {
    k: v for k, v in weights_default.items() if k in new_state_dict
}
new_state_dict.update(weights_default_process)
model.load_state_dict(new_state_dict)
model.to(device)

4. 训练

4.1 数据增强

为了防止过拟合,增加模型的泛化能力,我们会数据增强

python 复制代码
transform = transforms.Compose(
    [
        transforms.RandomRotation(45),  # 随机旋转,-45到45度之间随机选
        transforms.RandomCrop(32, padding=4),  # 随机裁剪
        transforms.RandomHorizontalFlip(p=0.5),  # 随机水平翻转 选择一个概率概率
        transforms.RandomVerticalFlip(p=0.5),  # 随机垂直翻转
        transforms.ToTensor(),
        transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2471, 0.2435, 0.2616)),
    ]
)

transformtest = transforms.Compose(
    [
        transforms.ToTensor(),
        transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2471, 0.2435, 0.2616)),
    ]
)

4.2 开始训练

python 复制代码
    # 损失函数和优化器
    loss_fn = nn.CrossEntropyLoss()
    optimizer = optim.Adam(model.parameters(), lr=lr)
    
    for epoch in range(epochs):
        # 开始时间
        start = time.time()
        # 总的损失值
        total_loss = 0.0
        # 样本数量:最后一次样本数量不是128
        samp_num = 0
        # 总的预测正确的分类
        correct = 0

        model.train()
        for i, (x, y) in enumerate(train_loader):
            x, y = x.to(device), y.to(device)
            # 累加样本数量
            samp_num += len(y)
            out = model(x)
            # 预测正确的样本数量
            correct += out.argmax(dim=1).eq(y).sum().item()
            loss = loss_fn(out, y)
            # 损失率累加
            total_loss += loss.item() * len(y)
            optimizer.zero_grad()
            loss.backward()
            optimizer.step()
            if i % 100 == 0:
                img_grid = torchvision.utils.make_grid(x)
                write1.add_image(
                    f"r_m_{epoch}_{i}", img_grid, epoch * len(train_loader) + i
                )

        print(
            "批次:%d 损失率:%.4f 准确率:%.4f 耗时:%.4f"
            % (epoch, total_loss / samp_num, correct / samp_num, time.time() - start)
        )
        # log metrics to wandb
        wandb.log({"acc": correct / samp_num, "loss": total_loss / samp_num})

4.3 保存模型

python 复制代码
torch.save(model.state_dict(), weightpath)

4.4 训练过程可视化

wandb

python 复制代码
 # 训练过程可视化
    wandb.init(
        project="my-qianyi-project",
        config={
            "learning_rate": lr,
            "architecture": "CNN",
            "dataset": "CIFAR-100",
            "batch_size": batzh_size,
            "epochs": epochs,
        },
    )

tensorboard

python 复制代码
write1 = SummaryWriter(log_dir=log_dir)
# 保存模型结构到tensorboard
write1.add_graph(model, input_to_model=torch.randn(1, 3, 32, 32).to(device=device))

5. 验证阶段

5.1 数据验证

python 复制代码
weights_default = torch.load(weightpath)
    # 再次获取resnet18原始神经网络并对齐fc层进行调整
    model = resnet18(pretrained=False)
    in_features = model.fc.in_features
    # 重写FC:我们这里做的是10分类
    model.fc = nn.Linear(in_features=in_features, out_features=10)
    model.load_state_dict(weights_default)
    model.to(device)
    model.eval()
    samp_num = 0
    correct = 0
    data2csv = np.empty(shape=(0, 13))
    for x, y in vaild_loader:
        x = x.to(device)
        y = y.to(device)
        # 累加样本数量
        samp_num += len(y)
        # 模型运算
        out = model(x)
        # 数组的合并
        data2csv = np.concatenate((data2csv, outdata_softmax), axis=0)
        # 预测正确的样本数量
        correct += out.argmax(dim=1).eq(y).sum().item()

    print("准确率:%.4f" % (correct / samp_num))

5.2 验证结果可视化

验证数据保存到Excel

python 复制代码
data2csv = np.empty(shape=(0, 13))

#数据整理
out = model(x)
outdata = out.cpu().detach()
outdata_softmax = torch.softmax(outdata, dim=1)
# 合并目标值到样本  [5, 7,9,0,1,,1,2,3,4,3,4]
outdata_softmax = np.concatenate(
    (
        # 本身预测的值
        outdata_softmax.numpy(),
        # 真正的目标值
        y.cpu().numpy().reshape(-1, 1),
        # 预测值
        outdata_softmax.argmax(dim=1).reshape(-1, 1),
        # 分类名称
        np.array([vaild_dataset.classes[i] for i in y.cpu().numpy()]).reshape(
            -1, 1
        ),
    ),
    axis=1,
)
# 数组的合并
data2csv = np.concatenate((data2csv, outdata_softmax), axis=0)

#写入CSV
columns = np.concatenate((vaild_dataset.classes, ["target", "prep", "分类"]))
pddata = pd.DataFrame(data2csv, columns=columns)
pddata.to_csv(csvpath, encoding="GB2312")

指标分析:可视化

python 复制代码
def analy():
    # 读取csv数据
    data1 = pd.read_csv(csvpath, encoding="GB2312")
    print(type(data1))
    # 整体数据分析报告
    report = classification_report(
        y_true=data1["target"].values,
        y_pred=data1["prep"].values,
    )
    print(report)
    # 准确度 Acc
    print(
        "准确度Acc:",
        accuracy_score(
            y_true=data1["target"].values,
            y_pred=data1["prep"].values,
        ),
    )
    # 精确度
    print(
        "精确度Precision:",
        precision_score(
            y_true=data1["target"].values, y_pred=data1["prep"].values, average="macro"
        ),
    )
    # 召回率
    print(
        "召回率Recall:",
        recall_score(
            # 100
            y_true=data1["target"].values,
            y_pred=data1["prep"].values,
            average="macro",
        ),
    )
    # F1 Score
    print(
        "F1 Score:",
        f1_score(
            y_true=data1["target"].values,
            y_pred=data1["prep"].values,
            average="macro",
        ),
    )
    pass


def matrix():
    # 读取csv数据
    data1 = pd.read_csv(csvpath, encoding="GB2312", index_col=0)
    confusion = confusion_matrix(
        # 0
        y_true=data1["target"].values,
        y_pred=data1["prep"].values,
        # labels=data1.columns[0:10].values,
    )
    print(confusion)
    # 绘制混淆矩阵
    plt.rcParams["font.sans-serif"] = ["SimHei"]
    plt.rcParams["axes.unicode_minus"] = False
    plt.matshow(confusion, cmap=plt.cm.Greens)
    plt.colorbar()
    for i in range(confusion.shape[0]):
        for j in range(confusion.shape[1]):
            plt.text(j, i, confusion[i, j], ha="center", va="center", color="b")
    plt.title("验证数据混淆矩阵")
    plt.xlabel("Predicted label")
    plt.xticks(range(10), data1.columns[0:10].values, rotation=45)
    plt.ylabel("True label")
    plt.yticks(range(10), data1.columns[0:10].values)
    plt.show()

6. 使用

python 复制代码
def app():
    dir = os.path.dirname(__file__)
    imgpath = os.path.join("./write", "6.png")
    # 读取图像文件 '8.png'
    img = cv2.imread(imgpath)
    # 将图像转换为灰度图
    img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
    # 对灰度图进行二值化处理,采用OTSU自适应阈值方法,并反转颜色
    ret, img = cv2.threshold(img, 0, 255, cv2.THRESH_BINARY_INV | cv2.THRESH_OTSU)
    plt.imshow(img)
    plt.show()
    # img = cv2.resize(img, (32, 32))
    img = torch.Tensor(img).unsqueeze(0)
    transform = transforms.Compose(
        [
            transforms.Resize((32, 32)),  # 调整输入图像大小为32x32
            transforms.ToTensor(),
            transforms.Normalize((0.1307,), (0.3081,)),
        ]
    )
    img = transform(img).unsqueeze(0)
    # 加载我们的模型
    net = LeNet5()
    net.load_state_dict(torch.load(modelpath))
    # 预测
    outputs = net(img)
    print(outputs)
    print(outputs.argmax(axis=1))
相关推荐
jndingxin3 分钟前
OpenCV 图形API(11)对图像进行掩码操作的函数mask()
人工智能·opencv·计算机视觉
Scc_hy12 分钟前
强化学习_Paper_1988_Learning to predict by the methods of temporal differences
人工智能·深度学习·算法
袁煦丞15 分钟前
【亲测】1.5万搞定DeepSeek满血版!本地部署避坑指南+内网穿透黑科技揭秘
人工智能·程序员·远程工作
大模型真好玩17 分钟前
理论+代码一文带你深入浅出MCP:人工智能大模型与外部世界交互的革命性突破
人工智能·python·mcp
遇码30 分钟前
大语言模型开发框架——LangChain
人工智能·语言模型·langchain·llm·大模型开发·智能体
在狂风暴雨中奔跑30 分钟前
使用AI开发Android界面
android·人工智能
飞哥数智坊32 分钟前
AI编程实战:30分钟实现Web 3D船舶航行效果
人工智能·three.js
誉鏐35 分钟前
从零开始设计Transformer模型(1/2)——剥离RNN,保留Attention
人工智能·深度学习·transformer
Ai野生菌36 分钟前
工具介绍 | SafeLLMDeploy教程来了 保护本地LLM安全部署
网络·人工智能·安全·大模型·llm
契合qht53_shine42 分钟前
OpenCV 从入门到精通(day_05)
人工智能·opencv·计算机视觉