12.6深度学习_模型优化和迁移_整体流程梳理

七、整体流程梳理

1. 引入使用的包

用到什么包,临时引入就可以,不用太担心。

python 复制代码
import time
import os

import numpy as np
import pandas as pd
import torch
import torch.nn as nn
import torch.optim as optim
import torchvision
import torchvision.transforms as transforms
from torch.utils.data import DataLoader
from torchvision.datasets import CIFAR10

from torchvision.models import resnet18, ResNet18_Weights
import wandb
from torch.utils.tensorboard import SummaryWriter
from sklearn.metrics import *
import matplotlib.pyplot as plt

2. 数据

python 复制代码
# 下面和以前就一样了
train_dataset = CIFAR10(
    root=datapath,
    train=True,
    download=True,
    transform=transform,
)
# 构建训练数据集
train_loader = DataLoader(
    #
    dataset=train_dataset,
    batch_size=batzh_size,
    shuffle=True,
    num_workers=2,
)

3. 模型

python 复制代码
# 再次获取resnet18原始神经网络并对齐fc层进行调整
model = resnet18(weights=None)

in_features = model.fc.in_features
# 重写FC:我们这里做的是10分类
model.fc = nn.Linear(in_features=in_features, out_features=10)

# 需要对权重信息进行处理:要加载我们训练之后最新的权重文件
weights_default = torch.load(weightpath)
weights_default.pop("fc.weight")
weights_default.pop("fc.bias")

# 把权重参数进行同步
new_state_dict = model.state_dict()
weights_default_process = {
    k: v for k, v in weights_default.items() if k in new_state_dict
}
new_state_dict.update(weights_default_process)
model.load_state_dict(new_state_dict)
model.to(device)

4. 训练

4.1 数据增强

为了防止过拟合,增加模型的泛化能力,我们会数据增强

python 复制代码
transform = transforms.Compose(
    [
        transforms.RandomRotation(45),  # 随机旋转,-45到45度之间随机选
        transforms.RandomCrop(32, padding=4),  # 随机裁剪
        transforms.RandomHorizontalFlip(p=0.5),  # 随机水平翻转 选择一个概率概率
        transforms.RandomVerticalFlip(p=0.5),  # 随机垂直翻转
        transforms.ToTensor(),
        transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2471, 0.2435, 0.2616)),
    ]
)

transformtest = transforms.Compose(
    [
        transforms.ToTensor(),
        transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2471, 0.2435, 0.2616)),
    ]
)

4.2 开始训练

python 复制代码
    # 损失函数和优化器
    loss_fn = nn.CrossEntropyLoss()
    optimizer = optim.Adam(model.parameters(), lr=lr)
    
    for epoch in range(epochs):
        # 开始时间
        start = time.time()
        # 总的损失值
        total_loss = 0.0
        # 样本数量:最后一次样本数量不是128
        samp_num = 0
        # 总的预测正确的分类
        correct = 0

        model.train()
        for i, (x, y) in enumerate(train_loader):
            x, y = x.to(device), y.to(device)
            # 累加样本数量
            samp_num += len(y)
            out = model(x)
            # 预测正确的样本数量
            correct += out.argmax(dim=1).eq(y).sum().item()
            loss = loss_fn(out, y)
            # 损失率累加
            total_loss += loss.item() * len(y)
            optimizer.zero_grad()
            loss.backward()
            optimizer.step()
            if i % 100 == 0:
                img_grid = torchvision.utils.make_grid(x)
                write1.add_image(
                    f"r_m_{epoch}_{i}", img_grid, epoch * len(train_loader) + i
                )

        print(
            "批次:%d 损失率:%.4f 准确率:%.4f 耗时:%.4f"
            % (epoch, total_loss / samp_num, correct / samp_num, time.time() - start)
        )
        # log metrics to wandb
        wandb.log({"acc": correct / samp_num, "loss": total_loss / samp_num})

4.3 保存模型

python 复制代码
torch.save(model.state_dict(), weightpath)

4.4 训练过程可视化

wandb

python 复制代码
 # 训练过程可视化
    wandb.init(
        project="my-qianyi-project",
        config={
            "learning_rate": lr,
            "architecture": "CNN",
            "dataset": "CIFAR-100",
            "batch_size": batzh_size,
            "epochs": epochs,
        },
    )

tensorboard

python 复制代码
write1 = SummaryWriter(log_dir=log_dir)
# 保存模型结构到tensorboard
write1.add_graph(model, input_to_model=torch.randn(1, 3, 32, 32).to(device=device))

5. 验证阶段

5.1 数据验证

python 复制代码
weights_default = torch.load(weightpath)
    # 再次获取resnet18原始神经网络并对齐fc层进行调整
    model = resnet18(pretrained=False)
    in_features = model.fc.in_features
    # 重写FC:我们这里做的是10分类
    model.fc = nn.Linear(in_features=in_features, out_features=10)
    model.load_state_dict(weights_default)
    model.to(device)
    model.eval()
    samp_num = 0
    correct = 0
    data2csv = np.empty(shape=(0, 13))
    for x, y in vaild_loader:
        x = x.to(device)
        y = y.to(device)
        # 累加样本数量
        samp_num += len(y)
        # 模型运算
        out = model(x)
        # 数组的合并
        data2csv = np.concatenate((data2csv, outdata_softmax), axis=0)
        # 预测正确的样本数量
        correct += out.argmax(dim=1).eq(y).sum().item()

    print("准确率:%.4f" % (correct / samp_num))

5.2 验证结果可视化

验证数据保存到Excel

python 复制代码
data2csv = np.empty(shape=(0, 13))

#数据整理
out = model(x)
outdata = out.cpu().detach()
outdata_softmax = torch.softmax(outdata, dim=1)
# 合并目标值到样本  [5, 7,9,0,1,,1,2,3,4,3,4]
outdata_softmax = np.concatenate(
    (
        # 本身预测的值
        outdata_softmax.numpy(),
        # 真正的目标值
        y.cpu().numpy().reshape(-1, 1),
        # 预测值
        outdata_softmax.argmax(dim=1).reshape(-1, 1),
        # 分类名称
        np.array([vaild_dataset.classes[i] for i in y.cpu().numpy()]).reshape(
            -1, 1
        ),
    ),
    axis=1,
)
# 数组的合并
data2csv = np.concatenate((data2csv, outdata_softmax), axis=0)

#写入CSV
columns = np.concatenate((vaild_dataset.classes, ["target", "prep", "分类"]))
pddata = pd.DataFrame(data2csv, columns=columns)
pddata.to_csv(csvpath, encoding="GB2312")

指标分析:可视化

python 复制代码
def analy():
    # 读取csv数据
    data1 = pd.read_csv(csvpath, encoding="GB2312")
    print(type(data1))
    # 整体数据分析报告
    report = classification_report(
        y_true=data1["target"].values,
        y_pred=data1["prep"].values,
    )
    print(report)
    # 准确度 Acc
    print(
        "准确度Acc:",
        accuracy_score(
            y_true=data1["target"].values,
            y_pred=data1["prep"].values,
        ),
    )
    # 精确度
    print(
        "精确度Precision:",
        precision_score(
            y_true=data1["target"].values, y_pred=data1["prep"].values, average="macro"
        ),
    )
    # 召回率
    print(
        "召回率Recall:",
        recall_score(
            # 100
            y_true=data1["target"].values,
            y_pred=data1["prep"].values,
            average="macro",
        ),
    )
    # F1 Score
    print(
        "F1 Score:",
        f1_score(
            y_true=data1["target"].values,
            y_pred=data1["prep"].values,
            average="macro",
        ),
    )
    pass


def matrix():
    # 读取csv数据
    data1 = pd.read_csv(csvpath, encoding="GB2312", index_col=0)
    confusion = confusion_matrix(
        # 0
        y_true=data1["target"].values,
        y_pred=data1["prep"].values,
        # labels=data1.columns[0:10].values,
    )
    print(confusion)
    # 绘制混淆矩阵
    plt.rcParams["font.sans-serif"] = ["SimHei"]
    plt.rcParams["axes.unicode_minus"] = False
    plt.matshow(confusion, cmap=plt.cm.Greens)
    plt.colorbar()
    for i in range(confusion.shape[0]):
        for j in range(confusion.shape[1]):
            plt.text(j, i, confusion[i, j], ha="center", va="center", color="b")
    plt.title("验证数据混淆矩阵")
    plt.xlabel("Predicted label")
    plt.xticks(range(10), data1.columns[0:10].values, rotation=45)
    plt.ylabel("True label")
    plt.yticks(range(10), data1.columns[0:10].values)
    plt.show()

6. 使用

python 复制代码
def app():
    dir = os.path.dirname(__file__)
    imgpath = os.path.join("./write", "6.png")
    # 读取图像文件 '8.png'
    img = cv2.imread(imgpath)
    # 将图像转换为灰度图
    img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
    # 对灰度图进行二值化处理,采用OTSU自适应阈值方法,并反转颜色
    ret, img = cv2.threshold(img, 0, 255, cv2.THRESH_BINARY_INV | cv2.THRESH_OTSU)
    plt.imshow(img)
    plt.show()
    # img = cv2.resize(img, (32, 32))
    img = torch.Tensor(img).unsqueeze(0)
    transform = transforms.Compose(
        [
            transforms.Resize((32, 32)),  # 调整输入图像大小为32x32
            transforms.ToTensor(),
            transforms.Normalize((0.1307,), (0.3081,)),
        ]
    )
    img = transform(img).unsqueeze(0)
    # 加载我们的模型
    net = LeNet5()
    net.load_state_dict(torch.load(modelpath))
    # 预测
    outputs = net(img)
    print(outputs)
    print(outputs.argmax(axis=1))
相关推荐
jndingxin4 分钟前
OpenCV 图形API(63)图像结构分析和形状描述符------计算图像中非零像素的边界框函数boundingRect()
人工智能·opencv·计算机视觉
旧故新长9 分钟前
支持Function Call的本地ollama模型对比评测-》开发代理agent
人工智能·深度学习·机器学习
微学AI22 分钟前
融合注意力机制和BiGRU的电力领域发电量预测项目研究,并给出相关代码
人工智能·深度学习·自然语言处理·注意力机制·bigru
知来者逆33 分钟前
计算机视觉——速度与精度的完美结合的实时目标检测算法RF-DETR详解
图像处理·人工智能·深度学习·算法·目标检测·计算机视觉·rf-detr
一勺汤36 分钟前
YOLOv11改进-双Backbone架构:利用双backbone提高yolo11目标检测的精度
人工智能·yolo·双backbone·double backbone·yolo11 backbone·yolo 双backbone
武汉唯众智创38 分钟前
高职人工智能技术应用专业(计算机视觉方向)实训室解决方案
人工智能·计算机视觉·人工智能实训室·计算机视觉实训室·人工智能计算机视觉实训室
Johny_Zhao1 小时前
MySQL 高可用集群搭建部署
linux·人工智能·mysql·信息安全·云计算·shell·yum源·系统运维·itsm
一只可爱的小猴子1 小时前
2022李宏毅老师机器学习课程笔记
人工智能·笔记·机器学习
地瓜机器人1 小时前
乐聚机器人与地瓜机器人达成战略合作,联合发布Aelos Embodied具身智能
人工智能·机器人
带娃的IT创业者1 小时前
《AI大模型趣味实战》基于RAG向量数据库的知识库AI问答助手设计与实现
数据库·人工智能