pytorch学习

m00—2024-08-30 19:39

1.安装及常用的命令

python+anacoda+pytorch下载

pytorch下载网站 （amd显卡下载cpuonly）

jupyter下载安装

2.python文件，python控制台，jupyter的区别

python文件是整个文件运行

python控制台是类似于matlab的命令行，可以运行一行回车，或者粘贴多行运行，

jupyter是将代码分块运行，本质都是同样的，.ipynnb文件后缀，默认保存到当前文件路径

3.dataset的使用

相关代码：

python 复制代码 
from torch.utils.data import Dataset
from PIL import Image
import os

class MyData(Dataset):

    def __init__(self, root_dir, label_dir):
        self.root_dir = root_dir    #根路径
        self.label_dir = label_dir  #标签路径
        self.path = os.path.join(self.root_dir, self.label_dir)   #完全的标签的相对路径(以当前系统的标准)
        self.img_path = os.listdir(self.path)         #列出当前路径下所有文件名，以列表形式存储

    def __getitem__(self, idx):
        img_name = self.img_path[idx]     #获得第几个图像
        img_item_path = os.path.join(self.root_dir, self.label_dir, img_name)   #完全的文件相对路径
        img = Image.open(img_item_path)      #打开指定的文件
        label = self.label_dir
        return img, label

    def __len__(self):
        return len(self.img_path)   #有多少个图像


root_dir = "datasets/hymenoptera_data/train"
# ants_label_dir = "ants"
# bees_label_dir = "bees"
# ants_dataset = MyData(root_dir, ants_label_dir)
# bees_dataset = MyData(root_dir, bees_label_dir)
# ants1, label1 = ants_dataset[1]
# ants2, label2 = ants_dataset[2]
# bees1, label3 = bees_dataset[1]
# ants1.show()
# train_dataset = ants_dataset+bees_dataset

target_dir = "bees_image"
img_path = os.listdir(os.path.join(root_dir, target_dir))
label = target_dir.split("_")[0]    #分割字符串，取前半部分
out_dir = "bees_label"
for i in img_path:
    file_name = i.split(".jpg")[0]
    with open(os.path.join(root_dir, out_dir, "{}.txt".format(file_name)), "w") as f:
        f.write(label)

4.TensorBoard和Transforms的使用

调取tensorboard报错：能进入浏览器但不显示线段，tensorboard版本过高

opencv使用清华源安装：

python 复制代码 
pip install opencv-python -i https://pypi.tuna.tsinghua.edu.cn/simple/

相关代码：

python 复制代码 
from torch.utils.tensorboard import SummaryWriter

writer = SummaryWriter("logs")


# writer.add_image()
for i in range(100):
    writer.add_scalar("y=3x", 3*i, i)

writer.close()

tag得不同才能识别为不同的文件

python 复制代码 
from torch.utils.tensorboard import SummaryWriter
import numpy as np
from PIL import Image

writer = SummaryWriter("logs")
img_path = "datasets/hymenoptera_data/train/ants_image/67270775_e9fdf77e9d.jpg"
img_PIL = Image.open(img_path)
img_array = np.array(img_PIL)  #类型转换


writer.add_image("train", img_array, 2, dataformats="HWC")   #numpy.ndarray

for i in range(100):
    writer.add_scalar("y=3x", 3*i, i)

writer.close()

ToTensor，Normalize，Resize，Compose，Randomcrop的使用：

python 复制代码 
from torch.utils.tensorboard import SummaryWriter
from PIL import Image
from torchvision import transforms

img_path = "datasets/hymenoptera_data/train/ants_image/67270775_e9fdf77e9d.jpg"


writer = SummaryWriter("logs")
img = Image.open(img_path)  #PIL格式
print(img)
#将PIL->tensor  ToTensor
tensor_trans = transforms.ToTensor()
tensor_img = tensor_trans(img)
writer.add_image("Tensor_img", tensor_img)   #numpy.ndarray


#Normalize  tensor->tensor
trans_norm = transforms.Normalize([0.5, 0.5, 0.5], [0.5, 0.5, 0.5]) #搅拌机
img_norm = trans_norm(tensor_img)    #将水果放入搅拌机内
writer.add_image("Normalize", img_norm)

#resize    PIL->PIL->tensor
trans_resize = transforms.Resize((400, 50))
img_resize = trans_resize(img)
print(img_resize)
img_resize = tensor_trans(img_resize)
writer.add_image("Resize_img", img_resize, 0)   #numpy.ndarray

trans_resize_2 = transforms.Resize(100)

#compose,裁剪和数据转换结合起来
trans_compose = transforms.Compose([trans_resize_2, tensor_trans])
img_resize_2 = trans_compose(img)
writer.add_image("Resize_img", img_resize_2, 1)

#randomcrop  s随机裁剪
trans_random = transforms.RandomCrop((400, 50))
trans_compose_2 = transforms.Compose([trans_random, tensor_trans])  #将多个函数组合起来，按顺序执行
for i in range(10):
    img_crop = trans_compose_2(img)
    writer.add_image("RandomCrop", img_crop, i)

writer.close()

5.dataloader的使用

python 复制代码 
import torchvision
from torch.utils.tensorboard import SummaryWriter
from torch.utils.data import DataLoader

datasets_transform = torchvision.transforms.Compose([
    torchvision.transforms.ToTensor()
])

test_set = torchvision.datasets.CIFAR10(root="./cfar10", train=False, transform=torchvision.transforms.ToTensor())
test_lodaer = DataLoader(dataset=test_set, batch_size=64, shuffle=True, num_workers=0, drop_last=False)    #batch_size 默认随机采样64个，不重复  drop_last=False保留最后一行不足64个图像的单元
                                                                                                           #shuffle=True保证两次采样的不同

writer = SummaryWriter("dataloader")
for epoch in range(2):
    step = 0
    for data in test_lodaer:
        imgs, targets = data
        writer.add_images("Epoch:{}".format(epoch), imgs, step)
        step = step + 1

writer.close()

images

6.nn.Module的使用

6.1 conv2d操作

代码：

python 复制代码 
import torch
import torchvision
from torch import nn
from torch.nn import Conv2d
from torch.utils.data import DataLoader
from torch.utils.tensorboard import SummaryWriter

dataset = torchvision.datasets.CIFAR10("./cfar10", train=False, transform=torchvision.transforms.ToTensor(), download=True)
dataloader = DataLoader(dataset, batch_size=64)

class Tudui(nn.Module):
    def __init__(self):
        super(Tudui, self).__init__()
        self.conv1 = Conv2d(in_channels=3, out_channels=6, kernel_size=3, stride=1, padding=0)


    def forward(self, x):
        x = self.conv1(x)
        return x

tudui = Tudui()
writer = SummaryWriter("./logs")
step = 0
for data in dataloader:
    imgs, targets = data
    output = tudui(imgs)
    # print(imgs.shape)    #torch.Size([64, 3, 32, 32]) DCHW
    # print(output.shape)  #torch.Size([64, 6, 30, 30])
    output = torch.reshape(output, (-1, 3, 30, 30))
    # print(output.shape)  # torch.Size([64, 6, 30, 30])
    writer.add_images("input", imgs, step)
    writer.add_images("output", output, step)
    step += 1

6.2 maxpool2d的使用

取最大值的操作

python 复制代码 
import torch
import torch.nn.functional as F
import torchvision
from torch import nn
from torch.nn import MaxPool2d
from torch.utils.data import DataLoader
from torch.utils.tensorboard import SummaryWriter

# input = torch.tensor([[1, 2, 0, 3, 1],
#                       [0, 1, 2, 3, 1],
#                       [1, 2, 1, 0, 0],
#                       [5, 2, 3, 1, 1],
#                       [2, 1, 0, 1, 1]],dtype=torch.float32)
#
# input = torch.reshape(input, (-1, 1, 5, 5))
dataset = torchvision.datasets.CIFAR10("./cfar10", train=False, transform=torchvision.transforms.ToTensor(), download=True)
dataloader = DataLoader(dataset, batch_size=64)

class Tudui(nn.Module):
    def __init__(self):
        super(Tudui, self).__init__()
        self.maxpool1 = MaxPool2d(kernel_size=3, ceil_mode=False)   #为True时是取边缘不足的地方

    def forward(self, input):
        output = self.maxpool1(input)
        return output


tudui = Tudui()
writer = SummaryWriter("logs_maxpool")
step = 0
for data in dataloader:
    imgs, targets = data
    writer.add_images("input", imgs, step)
    output = tudui(imgs)
    writer.add_images("output", output, step)
    step += 1

writer.close()

6.3 非线性激活函数的使用

python 复制代码 
import torch
import torchvision
from torch import nn
from torch.nn import Conv2d, ReLU, Sigmoid

# input = torch.tensor([[1, -0.5],
#                       [-1, 3]])
# input = torch.reshape(input, (-1, 1, 2, 2))

from torch.utils.data import DataLoader
from torch.utils.tensorboard import SummaryWriter

dataset = torchvision.datasets.CIFAR10("./cfar10", train=False, transform=torchvision.transforms.ToTensor(), download=True)
dataloader = DataLoader(dataset, batch_size=64)

class Tudui(nn.Module):
    def __init__(self):
        super(Tudui, self).__init__()
        self.relu1 = ReLU()   #i你place为true时替换当前的值，为False时赋给新的变量，一般为False
        self.sigmoid1 = Sigmoid()

    def forward(self, x):
        x = self.sigmoid1(x)
        return x

tudui = Tudui()
writer = SummaryWriter("./logs_relu")
step = 0
for data in dataloader:
    imgs, targets = data
    output = tudui(imgs)
    writer.add_images("input", imgs, step)
    writer.add_images("output", output, step)
    step += 1


writer.close()

6.4 线性层的使用

python 复制代码 
import torch
import torchvision
from torch import nn
from torch.nn import MaxPool2d, Linear
from torch.utils.data import DataLoader
from torch.utils.tensorboard import SummaryWriter

dataset = torchvision.datasets.CIFAR10("./cfar10", train=False, transform=torchvision.transforms.ToTensor(), download=True)
dataloader = DataLoader(dataset, batch_size=64, drop_last=True)

class Tudui(nn.Module):
    def __init__(self):
        super(Tudui, self).__init__()
        self.linear1 = Linear(196608, 10)   #为True时是取边缘不足的地方

    def forward(self, input):
        output = self.linear1(input)
        return output


tudui = Tudui()

for data in dataloader:
    imgs, targets = data
    print(imgs.shape)
    # output = torch.reshape(imgs, (1, 1, 1, -1))
    output = torch.flatten(imgs)
    print(output.shape)
    output = tudui(output)
    print(output.shape)

6.5 搭建神经网络

python 复制代码 
import torch
from torch import nn
from torch.nn import Conv2d, MaxPool2d, Flatten, Linear, Sequential
from torch.utils.tensorboard import SummaryWriter


class Tudui(nn.Module):
    def __init__(self):
        super(Tudui, self).__init__()
        self.model1 = Sequential(
            Conv2d(3, 32, 5, padding=2),
            MaxPool2d(2),
            Conv2d(32, 32, 5, padding=2),
            MaxPool2d(2),
            Conv2d(32, 64, 5, padding=2),
            MaxPool2d(2),
            Flatten(),
            Linear(1024, 64),
            Linear(64, 10)
        )

    def forward(self, x):
        x = self.model1(x)
        return x

tudui = Tudui()
print(tudui)
input = torch.ones((64, 3, 32, 32))
output = tudui(input)
print(output.shape)

writer = SummaryWriter("./logs_seq")
writer.add_graph(tudui,input)
writer.close()

6.5 损失函数和反向传播

根据损失函数为反向传播参数更新的依据

python 复制代码 
import torch
import torchvision
from torch import nn
from torch.nn import Conv2d, MaxPool2d, Flatten, Linear, Sequential
from torch.utils.data import DataLoader
from torch.utils.tensorboard import SummaryWriter

dataset = torchvision.datasets.CIFAR10("./cfar10", train=False, transform=torchvision.transforms.ToTensor(), download=True)
dataloader = DataLoader(dataset, batch_size=1)

class Tudui(nn.Module):
    def __init__(self):
        super(Tudui, self).__init__()
        self.model1 = Sequential(
            Conv2d(3, 32, 5, padding=2),
            MaxPool2d(2),
            Conv2d(32, 32, 5, padding=2),
            MaxPool2d(2),
            Conv2d(32, 64, 5, padding=2),
            MaxPool2d(2),
            Flatten(),
            Linear(1024, 64),
            Linear(64, 10)
        )

    def forward(self, x):
        x = self.model1(x)
        return x

loss = nn.CrossEntropyLoss()
tudui = Tudui()
optim = torch.optim.SGD(tudui.parameters(), lr=0.01)
for epoch in range(20):
    running_loss = 0
    for data in dataloader:
        imgs,targets = data
        outputs = tudui(imgs)
        result_loss = loss(outputs, targets)
        optim.zero_grad()
        result_loss.backward()
        optim.step()
        running_loss = running_loss+result_loss
    print(running_loss)

7.模型的修改与保存

save_demo.py

python 复制代码 
import torchvision
import torch

vgg16 = torchvision.models.vgg16(pretrained=False)
#保存方式1，模型结构+模型参数
torch.save(vgg16, "vgg16_method1.pth")

#保存方式2，模型参数（官方推荐）
torch.save(vgg16.state_dict(), "vgg16_method2.pth")

load_demo.py

python 复制代码 
import torch
import torchvision
from model_demo import *

#方式1，加载模型,存在一个陷阱，使用模型之前需要先声明，可以通过导入python文件进行解决
model = torch.load("tudui_method1.pth")

# print(model)

#方式2，加载模型
vgg16 = torchvision.models.vgg16(pretrained=False)
vgg16.load_state_dict(torch.load("vgg16_method2.pth"))
# print(vgg16)

model_demo.py

python 复制代码 
import torch
from torch import nn


class Tudui(nn.Module):
    def __init__(self):
        super(Tudui, self).__init__()
        self.conv1 = nn.Conv2d(3, 64, kernel_size=3)

    def forward(self, x):
        x = self.conv1(x)
        return x

tudui = Tudui()
torch.save(tudui, "tudui_method1.pth")

8.完整的训练套路

complete_demo.py

python 复制代码 
import torch
import torchvision
from torch import nn
from torch.nn import Conv2d, MaxPool2d, Flatten, Linear, Sequential
from torch.utils.data import DataLoader
from torch.utils.tensorboard import SummaryWriter
from model import *
#准备数据集
train_data = torchvision.datasets.CIFAR10("./cfar10", train=True, transform=torchvision.transforms.ToTensor(), download=True)
test_data = torchvision.datasets.CIFAR10("./cfar10", train=False, transform=torchvision.transforms.ToTensor(), download=True)


#数据集的长度
train_data_size = len(train_data)
test_data_size = len(test_data)

print("训练数据集的长度为:{}".format(train_data_size))
print("测试数据集的长度为:{}".format(test_data_size))

#利用dataloader来加载数据集
train_dataloader = DataLoader(train_data, batch_size=64)
test_dataloader = DataLoader(test_data, batch_size=64)


#创建模型
tudui = Tudui()

#损失函数
loss_fn = nn.CrossEntropyLoss()

#优化器
learning_rate = 1e-2
optimizer = torch.optim.SGD(tudui.parameters(), lr=learning_rate)

#设置训练网络的一些参数
#记录训练的次数
total_train_step = 0
#记录测试的次数
total_test_step = 0

#训练的轮数
epoch = 10

#添加tensorboard
writer = SummaryWriter("./logs")

for i in range(epoch):
    print("--------第{}轮训练开始--------".format(i+1))

    #训练步骤开始
    tudui.train()
    for data in train_dataloader:
        imgs, targets = data
        outputs = tudui(imgs)
        loss = loss_fn(outputs, targets)

        #优化器优化模型
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()

        total_train_step = total_train_step+1
        if total_train_step%100==0:
            print("训练次数：{}，Loss：{}".format(total_train_step, loss.item()))
            writer.add_scalar("train_loss", loss.item(), total_train_step)

    #测试步骤开始
    tudui.eval()
    total_test_loss = 0
    total_accuracy = 0
    with torch.no_grad():
        for data in test_dataloader:
            imgs, targets = data
            outputs = tudui(imgs)
            loss = loss_fn(outputs, targets)
            total_test_loss = total_test_loss + loss.item()
            accuracy = (outputs.argmax(1) == targets).sum()
            total_accuracy = total_accuracy + accuracy

    print("整体测试集上的Loss：{}".format(total_test_loss))
    print("整体测试集上的正确率：{}".format(total_accuracy/test_data_size))
    writer.add_scalar("test_loss", total_test_loss, total_test_step)
    writer.add_scalar("test_accuracy", total_accuracy/test_data_size, total_test_step)
    total_test_step = total_test_step+1

    torch.save(tudui, "tudui_{}.pth".format(i))
    print("模型已保存")
writer.close()

model.py

python 复制代码 
import torch
from torch import nn
from torch.nn import Conv2d, MaxPool2d, Flatten, Linear, Sequential

class Tudui(nn.Module):
    def __init__(self):
        super(Tudui, self).__init__()
        self.model1 = Sequential(
            Conv2d(3, 32, 5, padding=2),
            MaxPool2d(2),
            Conv2d(32, 32, 5, padding=2),
            MaxPool2d(2),
            Conv2d(32, 64, 5, padding=2),
            MaxPool2d(2),
            Flatten(),
            Linear(1024, 64),
            Linear(64, 10)
        )

    def forward(self, x):
        x = self.model1(x)
        return x


if __name__ == '__main__':
    tudui = Tudui()
    input = torch.ones((64, 3, 32, 32))
    output = tudui(input)
    print(output.shape)

9.利用GPU训练

python 复制代码 
import torch
import torchvision
from torch import nn
from torch.nn import Conv2d, MaxPool2d, Flatten, Linear, Sequential
from torch.utils.data import DataLoader
from torch.utils.tensorboard import SummaryWriter

import time
#准备数据集
train_data = torchvision.datasets.CIFAR10("./cfar10", train=True, transform=torchvision.transforms.ToTensor(), download=True)
test_data = torchvision.datasets.CIFAR10("./cfar10", train=False, transform=torchvision.transforms.ToTensor(), download=True)


#数据集的长度
train_data_size = len(train_data)
test_data_size = len(test_data)

print("训练数据集的长度为:{}".format(train_data_size))
print("测试数据集的长度为:{}".format(test_data_size))

#利用dataloader来加载数据集
train_dataloader = DataLoader(train_data, batch_size=64)
test_dataloader = DataLoader(test_data, batch_size=64)

#定义使用的设备
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")

#创建模型
class Tudui(nn.Module):
    def __init__(self):
        super(Tudui, self).__init__()
        self.model1 = Sequential(
            Conv2d(3, 32, 5, padding=2),
            MaxPool2d(2),
            Conv2d(32, 32, 5, padding=2),
            MaxPool2d(2),
            Conv2d(32, 64, 5, padding=2),
            MaxPool2d(2),
            Flatten(),
            Linear(1024, 64),
            Linear(64, 10)
        )

    def forward(self, x):
        x = self.model1(x)
        return x
tudui = Tudui()
tudui = tudui.to(device)
#损失函数
loss_fn = nn.CrossEntropyLoss()
loss_fn = loss_fn.to(device)
#优化器
learning_rate = 1e-2
optimizer = torch.optim.SGD(tudui.parameters(), lr=learning_rate)

#设置训练网络的一些参数
#记录训练的次数
total_train_step = 0
#记录测试的次数
total_test_step = 0

#训练的轮数
epoch = 10

#添加tensorboard
writer = SummaryWriter("./logs")
start_time = time.time()
for i in range(epoch):
    print("--------第{}轮训练开始--------".format(i+1))

    #训练步骤开始
    tudui.train()
    for data in train_dataloader:
        imgs, targets = data
        imgs = imgs.to(device)
        targets = targets.to(device)
        outputs = tudui(imgs)
        loss = loss_fn(outputs, targets)

        #优化器优化模型
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()

        total_train_step = total_train_step+1
        if total_train_step%100==0:
            end_time = time.time()
            print(end_time-start_time)
            print("训练次数：{}，Loss：{}".format(total_train_step, loss.item()))
            writer.add_scalar("train_loss", loss.item(), total_train_step)

    #测试步骤开始
    tudui.eval()
    total_test_loss = 0
    total_accuracy = 0
    with torch.no_grad():
        for data in test_dataloader:
            imgs, targets = data
            imgs = imgs.to(device)
            targets = targets.to(device)
            outputs = tudui(imgs)
            loss = loss_fn(outputs, targets)
            total_test_loss = total_test_loss + loss.item()
            accuracy = (outputs.argmax(1) == targets).sum()
            total_accuracy = total_accuracy + accuracy

    print("整体测试集上的Loss：{}".format(total_test_loss))
    print("整体测试集上的正确率：{}".format(total_accuracy/test_data_size))
    writer.add_scalar("test_loss", total_test_loss, total_test_step)
    writer.add_scalar("test_accuracy", total_accuracy/test_data_size, total_test_step)
    total_test_step = total_test_step+1

    torch.save(tudui, "tudui_{}.pth".format(i))
    print("模型已保存")
writer.close()

10.完整的验证套路

训练模型：来去得到模型

python 复制代码 
import torch
import torchvision.transforms
from PIL import Image
from torch import nn
from torch.nn import Conv2d, MaxPool2d, Flatten, Linear, Sequential
img_path = "./data/dog1.png"
image = Image.open(img_path)
print(image)
transform = torchvision.transforms.Compose([torchvision.transforms.Resize((32, 32)),
                                            torchvision.transforms.ToTensor()])

image = transform(image)


class Tudui(nn.Module):
    def __init__(self):
        super(Tudui, self).__init__()
        self.model = Sequential(
            Conv2d(3, 32, 5, 1, 2),
            MaxPool2d(2),
            Conv2d(32, 32, 5, 1, 2),
            MaxPool2d(2),
            Conv2d(32, 64, 5, 1, 2),
            MaxPool2d(2),
            Flatten(),
            Linear(1024, 64),
            Linear(64, 10)
        )

    def forward(self, x):
        x = self.model(x)
        return x


model = torch.load("tudui_9.pth", map_location=torch.device('cpu'))
print(model)
image = torch.reshape(image, (-1, 3, 32, 32))
model.eval()
with torch.no_grad():
    output = model(image)
相关推荐
EterNity_TiMe_1 小时前
【论文复现】（CLIP）文本也能和图像配对
python·学习·算法·性能优化·数据分析·clip
sanguine__1 小时前
java学习-集合
学习
lxlyhwl1 小时前
【STK学习】part2-星座-目标可见性与覆盖性分析
学习
nbsaas-boot1 小时前
如何利用ChatGPT加速开发与学习：以BPMN编辑器为例
学习·chatgpt·编辑器
CV学术叫叫兽2 小时前
一站式学习：害虫识别与分类图像分割
学习·分类·数据挖掘
我们的五年2 小时前
【Linux课程学习】：进程程序替换，execl，execv，execlp，execvp，execve，execle，execvpe函数
linux·c++·学习
一棵开花的树，枝芽无限靠近你2 小时前
【PPTist】添加PPT模版
前端·学习·编辑器·html
VertexGeek2 小时前
Rust学习（八）：异常处理和宏编程：
学习·算法·rust
二进制_博客3 小时前
Flink学习连载文章4-flink中的各种转换操作
大数据·学习·flink
codebolt3 小时前
ADS学习记录
学习
热门推荐
01(欧拉)openEuler系统添加网卡文件配置流程、(欧拉)openEuler系统手动配置ipv6地址流程、(欧拉)openEuler系统网络管理说明02PyTorch机器学习实现液态神经网络03【HarmonyOS】HUAWEI DevEco Studio 下载地址汇总04玄机平台应急响应—webshell查杀05Coze扣子平台完整体验和实践（附国内和国际版对比）06Ubuntu 20.04使用Livox mid 360 测试 FAST_LIO07【目标跟踪】相机运动补偿08Unity中PICO实现 隔空取物 和 接触抓取物体09RAG 实践- Ollama+RagFlow 部署本地知识库10量化方法怎么选？如何评估量化后的大模型LLM？