使用虚线的残差块类：（yolo v3版本，一次成功！）
封装如下：
global void shortcut_kernel(int size, int minw, int minh, int minc, int stride, int sample, int batch, int w1, int h1, int c1, float* add, int w2, int h2, int c2, float* out)
{
int id = (blockIdx.x + blockIdx.y * gridDim.x) * blockDim.x + threadIdx.x;
if (id >= size) return;
int i = id % minw;
id /= minw;
int j = id % minh;
id /= minh;
int k = id % minc;
id /= minc;
int b = id % batch;
int out_index = i * sample + w2 * (j * sample + h2 * (k + c2 * b));
int add_index = i * stride + w1 * (j * stride + h1 * (k + c1 * b));
outout_index += addadd_index;
}
void shortcut_gpu(int batch, int w1, int h1, int c1, float* add, int w2, int h2, int c2, float* out)
{
int minw = (w1 < w2) ? w1 : w2;
int minh = (h1 < h2) ? h1 : h2;
int minc = (c1 < c2) ? c1 : c2;
int stride = w1 / w2;
int sample = w2 / w1;
// assert(stride == h1 / h2);
// assert(sample == h2 / h1);
if (stride < 1) stride = 1;
if (sample < 1) sample = 1;
int size = batch * minw * minh * minc;
//shortcut_kernel << <cuda_gridsize(size), BLOCK, 0, get_cuda_stream() >> >(size, minw, minh, minc, stride, sample, batch, w1, h1, c1, add, w2, h2, c2, out);
shortcut_kernel << <(size + 255) / 256, 256 >> > (size, minw, minh, minc, stride, sample, batch, w1, h1, c1, add, w2, h2, c2, out);
error_handling(cudaPeekAtLastError());
}
class xuxianResidual2:public Layer {//输入是64*32*32，输出是128*16*16,chw->2*c*h/2*w/2，这个虚线版本未处理bn和leaky relu
public:
xuxianResidual2(cudnnHandle_t& cudnn_, int batch_, int c, int h, int w) : cudnn(cudnn_), batch(batch_)
, _c(c), _h(h), _w(w) {
layers.emplace_back(std::make_shared<Conv2D>(cudnn, batch, _c, 2 * _c, _h, _w, 3, 2, 1));//c3,6*12*12->>16*8*8
layers.emplace_back(std::make_shared<BN>(cudnn, batch, 2 * _c, _h / 2, _w / 2));
layers.emplace_back(std::make_shared<LeakyRL>(cudnn, batch, 2 * _c, _h / 2, _w / 2)); //c3,6*12*12->>16*8*8
layers.emplace_back(std::make_shared<Conv2D>(cudnn, batch, 2 * _c, 2 * _c, _h / 2, _w / 2, 3, 1, 1));
layers.emplace_back(std::make_shared<BN>(cudnn, batch, 2 * _c, _h / 2, _w / 2));
layers.emplace_back(std::make_shared<LeakyRL>(cudnn, batch, 2 * _c, _h / 2, _w / 2));//20260710收到darknet的启发1506
cudaMalloc(&output, batch * _c * _h * _w / 2 * sizeof(float));//输出32*32*32-----------------------显然输入也是32*32*32
cudaMalloc(&input2, batch * _c * _h * _w * sizeof(float));
//cudaMalloc(&input3, batch * _c * _h * _w * sizeof(float));
cudaMalloc(&d_residual, batch * _c * _h * _w * sizeof(float));
// cudaMalloc(&output, batch * 10 * sizeof(float));//这里的10代表10个类，所以不能用
cudaMalloc(&grad_input, batch * _c * _h * _w * sizeof(float));//反向和梯度计算不管！！！！！！！！！！！！！！
//这里的类cbl2的权重参数加载和保存先不管！202609110752
//(cudnnHandle_t& cudnn_, int batch_, int ci, int co, int h, int w, int juanjihe, int pading=0)
//newX = new CBL2(cudnn, batch, _c, 2 * _c, _h, _w, 1, 0);//这个要用cbl，或者重新封装一个con+bn,使用cbl2，stride=2，pad=0，卷积核=1*1
//这里的残差全部在leaky relu之后相加！202609110753
}
void forward(float* input_)override {
//在此，应该有一个
// std::make_shared<Conv2D>(cudnn, batch, _c, 2 * _c, _h, _w, 1, 2, 1);
//std::make_shared<BN>(cudnn, batch, 2 * _c, _h / 2, _w / 2);//先不管bn层
//cudaMemcpy(input3, input_, sizeof(float) * batch * _c * _h * _w, cudaMemcpyDeviceToDevice);
//newX->forward(input_);
input = input_;
input2 = input_;
//input = newX->get_output();//不对，这个应该是残差相加的X
//input2 = newX->get_output();
for (const auto& l : layers) {
l->forward(input);
input = l->get_output();//这里的input与input2的nchw相同
}
int NN = batch * 2 * _c * _h / 2 * _w / 2;
//residual_forward_kernel << <(NN + 255) / 256, 256 >> > (output, input, input2, NN);//这里不对也要改，input2是ncwh的！
shortcut_gpu(batch, _w, _h, _c, input2, _w / 2, _h / 2, _c * 2, input);
cudaMemcpy(output, input, sizeof(float) * NN, cudaMemcpyDeviceToDevice);
error_handling(cudaGetLastError());
}
void forward2(float* input_)override {//这里的虚线残差比lenet中复杂，w，h不一样，c也不一样！202609110724
// cudaMemcpy(input3, input_, sizeof(float) * batch * _c * _h * _w, cudaMemcpyDeviceToDevice);
//newX->forward(input_);
input = input_;
input2 = input_;
/*input2 = newX->get_output();*/
for (const auto& l : layers) {
l->forward2(input);
input = l->get_output();
}
int NN = batch * _c * _h * _w / 2;
//residual_forward_kernel << <(NN + 255) / 256, 256 >> > (output, input, input2, NN);//这里不对也要改，input2是ncwh的！
shortcut_gpu(batch, _w, _h, _c, input2, _w / 2, _h / 2, _c * 2, input);
cudaMemcpy(output, input, sizeof(float) * NN, cudaMemcpyDeviceToDevice);
error_handling(cudaGetLastError());
}
void backward(float* grad_output)override {//虚线反向传播明天继续！202609101937
float* grad = grad_output;// batch, 2 * _c, _h / 2, _w / 2
float* grad备用 = grad_output;
////对应代码：newX->forward(input_);
////float* grad备用 = grad_output;
//newX->backward(grad_output);//1*1的left也要这样处理
//float* grad备用 = newX->get_grad_input();// batch, _c, _h , _w
for (int i = layers.size() - 1; i >= 0; i--) {
layersi->backward(grad);
grad = layersi->get_grad_input();
}// batch, _c, _h , _w ,正路已经到达，
int NN = batch * _c * _h * _w/2;
int threads = 256;
int blocks = (NN + threads - 1) / threads;
//使用yolo 的残差试一试，看两个bn有什么情况
mul<< <blocks, threads >> > (grad备用, output, d_residual, NN);//这里不对也要改，input2是ncwh的！不再使用input2，使用output
error_handling(cudaGetLastError());
//shortcut_gpu(batch, _w, _h, _c, d_residual, _w, _h, _c, grad);//虚线l.out_c=12,l.c=16,在这里是实线，l.out_c=16,l.c=16
shortcut_gpu(batch, _w/2, _h/2, _c*2, d_residual, _w, _h, _c, grad);
cudaMemcpy(grad_input, grad, sizeof(float) * NN, cudaMemcpyDeviceToDevice);
error_handling(cudaGetLastError());//仍然是第二个bn层方差均值为零
}
int getname() override { return 45; }
float* get_output() override { return output; }
float* get_grad_input() override { return grad_input; }
void update(float lr) {
for (const auto& l : layers) {
l->update(lr);
}
}
~xuxianResidual2() {
cudaFree(output);
cudaFree(grad_input);
}
private:
// cublasHandle_t &cublas;
int _c, _h, _w;
cudnnHandle_t& cudnn;
int batch;
float* input, * output, * grad_input;
float* input2; //float* input3;
float* d_residual;
public:
std::vector<std::shared_ptr<Layer>> layers;
// CBL2* newX;
};
我的残差块和yolo v3调用的核函数一样，但参数不同！使用环境也不同！特别是backward中，你看yolo v3：
cindex = aindex * (bindex > 0 ? 1 : 0.1);
//说明leakyrelu放在add之后，即out +=x，out =relu（out）
而我的是： cindex = aindex * bindex ;//即out =relu（out），out +=x
yolo v3使用 输出**：mul << <blocks, threads >> > (grad备用,** output**, d_residual, NN);**
我的是用 输入**：mul << <blocks, threads >> > (grad备用,** input2**, d_residual, NN);**
backward中执行完mul后：
yolo v3是：shortcut_gpu(batch, _w/2, _h/2, _c*2, d_residual, _w, _h, _c, grad);
我的是 ：shortcut_gpu(batch, _w, _h, _c, d_residual, _w, _h, _c, grad);
：在forward中，yolo v3仍然使用：shortcut_gpu函数
而我使用的是：global void residual_forward_kernel**(float* out, const float* inp1, const float* inp2, int N) {
int idx = blockIdx.x * blockDim.x + threadIdx.x;
if (idx < N) {
outidx = inp1idx + inp2idx;
}
}**
总体来说，还是yolo v3的好，简洁，弹性好，不宜读懂，工程用的多！我的太单纯，适应性差！但我也是从公式一步一步推出来，从cpu一直写到c++ cudnn（gpu），验证过来的！适合学习！
yolo v3版 虚线残差架构如下**：**
//resnet18//resnet18//resnet18
layers.emplace_back(std::make_shared<Conv2D>(cudnn, batch, 5, 32, 32, 32, 3, 1, 1));
//第一残差开始
layers.emplace_back(std::make_shared<residualExt22>(cudnn, batch, 32, 32, 32));
//第二残差开始
layers.emplace_back(std::make_shared<Conv2D>(cudnn, batch, 32, 64, 32, 32, 3, 1, 1));
layers.emplace_back(std::make_shared<residualExt22>(cudnn, batch, 64, 32, 32));
//第三残差开始，虚线
//xuxianResidual，使用yolo v3方式
layers.emplace_back(std::make_shared< xuxianResidual2**>(cudnn, batch, 64, 32, 32));**
//第四残差开始
layers.emplace_back(std::make_shared<residualExt22>(cudnn, batch, 128, 16, 16));
layers.emplace_back(std::make_shared<Conv2D>(cudnn, batch, 128, 256, 16, 16, 3, 1, 1));
//第六残差开始
layers.emplace_back(std::make_shared<residualExt22>(cudnn, batch, 256, 16, 16));
layers.emplace_back(std::make_shared<averPool2D>(cudnn, batch, 256, 16,16, 2, 2, 0, 2));
layers.emplace_back(std::make_shared<Linear>(cublas, batch, 256 * 64, 384));
layers.emplace_back(std::make_shared<BN>(cudnn, batch, 384, 1, 1));
layers.emplace_back(std::make_shared<LeakyRL>(cudnn, batch, 384, 1, 1));
layers.emplace_back(std::make_shared<Dropout>(cudnn, batch, 384, 1, 1));
layers.emplace_back(std::make_shared<Linear>(cublas, batch, 384, 10));
成绩如下：
轮次：0
learn rate：0.001
时间: 26315.244141 ms
train Classification result: 10.11% ok (used 49984 images)
时间: 2336.856934 ms
Test Classification result: 10.00% ok (used 9984 images)
轮次：1
learn rate：0.001
时间: 30842.468750 ms
train Classification result: 9.91% ok (used 49984 images)
时间: 2343.500000 ms
Test Classification result: 9.99% ok (used 9984 images)
轮次：2
learn rate：0.001
时间: 30982.064453 ms
train Classification result: 10.07% ok (used 49984 images)
时间: 2347.152100 ms
Test Classification result: 9.99% ok (used 9984 images)
轮次：3
learn rate：0.001
时间: 31105.507812 ms
train Classification result: 9.87% ok (used 49984 images)
时间: 2341.756104 ms
Test Classification result: 10.00% ok (used 9984 images)
轮次：4
learn rate：0.001
时间: 31147.496094 ms
train Classification result: 10.20% ok (used 49984 images)
时间: 2337.253906 ms
Test Classification result: 10.00% ok (used 9984 images)
轮次：5
learn rate：0.001
时间: 31208.203125 ms
train Classification result: 9.97% ok (used 49984 images)
时间: 2351.656982 ms
Test Classification result: 10.00% ok (used 9984 images)
轮次：6
learn rate：0.001
时间: 31214.199219 ms
train Classification result: 9.95% ok (used 49984 images)
时间: 2346.457031 ms
Test Classification result: 10.00% ok (used 9984 images)
轮次：7
learn rate：0.001
时间: 31260.355469 ms
train Classification result: 10.10% ok (used 49984 images)
时间: 2354.116943 ms
Test Classification result: 10.00% ok (used 9984 images)
轮次：8
learn rate：0.001
时间: 31823.050781 ms
train Classification result: 10.10% ok (used 49984 images)
时间: 2504.180908 ms
Test Classification result: 10.00% ok (used 9984 images)
轮次：9
learn rate：0.001
时间: 31878.787109 ms
train Classification result: 10.30% ok (used 49984 images)
时间: 2501.122070 ms
Test Classification result: 9.98% ok (used 9984 images)
轮次：10
learn rate：0.001
时间: 31954.679688 ms
train Classification result: 11.16% ok (used 49984 images)
时间: 2489.648926 ms
Test Classification result: 10.01% ok (used 9984 images)
轮次：11
learn rate：0.001
时间: 31929.302734 ms
train Classification result: 12.20% ok (used 49984 images)
时间: 2484.318115 ms
Test Classification result: 9.99% ok (used 9984 images)
轮次：12
learn rate：0.001
时间: 31987.031250 ms
train Classification result: 13.74% ok (used 49984 images)
时间: 2461.681885 ms
Test Classification result: 10.01% ok (used 9984 images)
轮次：13
learn rate：0.001
时间: 32047.892578 ms
train Classification result: 16.14% ok (used 49984 images)
时间: 2491.000000 ms
Test Classification result: 10.02% ok (used 9984 images)
轮次：14
learn rate：0.001
rb均值: 2.5410492420,rb方差:17.679039001465
时间: 31994.019531 ms
train Classification result: 19.79% ok (used 49984 images)
时间: 2498.277100 ms
Test Classification result: 10.00% ok (used 9984 images)
轮次：15
learn rate：0.001
rb均值: 2.0152964592,rb方差:13.053104400635
rb均值: 6.6344642639,rb方差:37.034439086914
时间: 31959.388672 ms
train Classification result: 25.59% ok (used 49984 images)
时间: 2466.862061 ms
Test Classification result: 9.99% ok (used 9984 images)
轮次：16
learn rate：0.0001
rb均值: 1.9933570623,rb方差:16.229248046875
rb均值: 7.8297410011,rb方差:41.423637390137
时间: 31225.947266 ms
train Classification result: 43.14% ok (used 49984 images)
时间: 2391.681885 ms
Test Classification result: 10.02% ok (used 9984 images)
轮次：17
learn rate：0.0001
rb均值: 1.6065976620,rb方差:16.059625625610
rb均值: 8.0434274673,rb方差:43.739288330078
时间: 31233.816406 ms
train Classification result: 46.28% ok (used 49984 images)
时间: 2376.014893 ms
Test Classification result: 10.01% ok (used 9984 images)
轮次：18
learn rate：0.0001
rb均值: 1.6071808338,rb方差:13.767983436584
rb均值: 8.3136625290,rb方差:45.571079254150
时间: 31227.597656 ms
train Classification result: 48.51% ok (used 49984 images)
时间: 2382.270996 ms
Test Classification result: 9.99% ok (used 9984 images)
轮次：19
learn rate：0.0001
rb均值: 1.8572018147,rb方差:15.183853149414
rb均值: 8.6042804718,rb方差:48.299381256104
时间: 31924.015625 ms
train Classification result: 49.62% ok (used 49984 images)
时间: 2494.606934 ms
Test Classification result: 10.00% ok (used 9984 images)
轮次：20
learn rate：0.0001
rb均值: 1.9974590540,rb方差:15.955725669861
rb均值: 8.8632125854,rb方差:51.546028137207
时间: 31232.248047 ms
train Classification result: 51.38% ok (used 49984 images)
时间: 2500.370117 ms
Test Classification result: 10.01% ok (used 9984 images)
轮次：21
learn rate：0.0001
rb均值: 2.1330072880,rb方差:16.448982238770
rb均值: 9.0661144257,rb方差:53.910503387451
时间: 31220.072266 ms
train Classification result: 52.68% ok (used 49984 images)
时间: 2367.571045 ms
Test Classification result: 10.01% ok (used 9984 images)
轮次：22
learn rate：0.0001
rb均值: 2.3347156048,rb方差:18.674993515015
rb均值: 9.3070430756,rb方差:56.158473968506
时间: 31239.619141 ms
train Classification result: 53.89% ok (used 49984 images)
时间: 2403.011963 ms
Test Classification result: 9.99% ok (used 9984 images)
轮次：23
learn rate：0.0001
rb均值: 6.6057753563,rb方差:5.208174705505
rb均值: -1.1543070078,rb方差:5.200778484344
rb均值: 2.4746243954,rb方差:18.864133834839
rb均值: 9.4319963455,rb方差:59.018497467041
时间: 31226.394531 ms
train Classification result: 54.71% ok (used 49984 images)
时间: 2385.482910 ms
Test Classification result: 10.02% ok (used 9984 images)
轮次：24
learn rate：0.0001
rb均值: 1.3367358446,rb方差:5.181817054749
rb均值: 6.8033924103,rb方差:5.574217796326
rb均值: -0.7874052525,rb方差:6.068223476410
rb均值: 2.4079535007,rb方差:19.849925994873
rb均值: 9.5169029236,rb方差:61.158596038818
时间: 32023.800781 ms
train Classification result: 56.09% ok (used 49984 images)
时间: 2552.230957 ms
Test Classification result: 10.00% ok (used 9984 images)
轮次：25
learn rate：1e-05
rb均值: 1.3546354771,rb方差:5.144558429718
rb均值: 6.8489518166,rb方差:5.577424526215
rb均值: -0.8116450906,rb方差:6.143618583679
rb均值: 2.3947772980,rb方差:19.468599319458
rb均值: 9.5320358276,rb方差:62.456512451172
时间: 31992.714844 ms
train Classification result: 58.59% ok (used 49984 images)
时间: 2526.507080 ms
Test Classification result: 69.16% ok (used 9984 images)
轮次：26
learn rate：1e-05
rb均值: 1.3689221144,rb方差:5.140226840973
rb均值: 6.8686351776,rb方差:5.571580886841
rb均值: -0.9068864584,rb方差:6.091242313385
rb均值: 2.4028677940,rb方差:19.873777389526
rb均值: 9.5522556305,rb方差:62.975917816162
时间: 32024.205078 ms
train Classification result: 58.49% ok (used 49984 images)
时间: 2392.690918 ms
Test Classification result: 68.68% ok (used 9984 images)
轮次：27
learn rate：1e-05
rb均值: 1.3530678749,rb方差:5.070459365845
rb均值: 6.8881807327,rb方差:5.630280494690
rb均值: -1.0320429802,rb方差:6.050611972809
rb均值: 2.4233353138,rb方差:19.623939514160
rb均值: 9.5934524536,rb方差:62.836025238037
时间: 31219.636719 ms
train Classification result: 58.72% ok (used 49984 images)
时间: 2395.064941 ms
Test Classification result:66.71% ok (used 9984 images)
轮次：28
learn rate：1e-05
rb均值: 1.3530298471,rb方差:5.130559444427
rb均值: 6.9049024582,rb方差:5.666448116302
rb均值: -1.0009651184,rb方差:6.203491687775
rb均值: 2.4050171375,rb方差:19.104766845703
rb均值: 9.5965929031,rb方差:63.030498504639
时间: 31231.027344 ms
train Classification result: 58.80% ok (used 49984 images)
时间: 2393.217041 ms
Test Classification result:67.34% ok (used 9984 images)
奇葩吧！test还是大于train！
lr调整如下：lr=0.001开始
if (chengji0 >= 20)lr = 0.0001;//方差还没有异常下手，监测训练成绩
//if (chengji0 >= 35)lr = 0.0001;//17次结果，方差开始异动
if (chengji0 >= 55)lr = 0.00001;//19次结束，0也算一次，实际是20
虽然pytorch resnet18还没有完全实现代替，但还是收获很多！很值得！