之前的训练cifar10的lenet网络架构：
修改后的网络架构：
看到没？残差后的relu被我注释掉了！
我们再看前版本残差块residualExt2：
再看新版本residualExt2改了什么：
class residualExt2 :public Layer {//改进成先降维，再升维202607101844
public:
residualExt2(cudnnHandle_t& cudnn_, int batch_, int c, int h, int w) : cudnn(cudnn_), batch(batch_)
, _c(c), _h(h), _w(w) {
layers.emplace_back(std::make_shared<Conv2D>(cudnn, batch, _c, _c , _h, _w, 1, 1));//c3,6*12*12->>16*8*8
layers.emplace_back(std::make_shared<BN>(cudnn, batch, _c , _h, _w));
layers.emplace_back(std::make_shared<LeakyRL>(cudnn, batch, _c , _h, _w)); //c3,6*12*12->>16*8*8
layers.emplace_back(std::make_shared<Conv2D>(cudnn, batch, _c , _c, _h, _w, 3, 1, 1));
//尝试残差，此处要记住输入X
//layers.emplace_back(std::make_shared<Conv2D>(cudnn, batch, _c, _c/2, _h, _w, 1, 1));//c3,6*12*12->>16*8*8
//layers.emplace_back(std::make_shared<BN>(cudnn, batch, _c/2, _h, _w));
//layers.emplace_back(std::make_shared<LeakyRL>(cudnn, batch, _c /2, _h, _w)); //c3,6*12*12->>16*8*8
//layers.emplace_back(std::make_shared<Conv2D>(cudnn, batch, _c/2, _c, _h, _w, 3, 1, 1));
layers.emplace_back(std::make_shared<BN>(cudnn, batch, _c, _h, _w));
layers.emplace_back(std::make_shared<LeakyRL>(cudnn, batch, _c, _h, _w));//20260710收到darknet的启发1506
cudaMalloc(&output, batch * _c * _h * _w * sizeof(float));//输出32*32*32-----------------------显然输入也是32*32*32
cudaMalloc(&input2, batch * _c * _h * _w * sizeof(float));
cudaMalloc(&d_residual, batch * _c * _h * _w * sizeof(float));
// cudaMalloc(&output, batch * 10 * sizeof(float));//这里的10代表10个类，所以不能用
cudaMalloc(&grad_input, batch * _c * _h * _w * sizeof(float));//反向和梯度计算不管！！！！！！！！！！！！！！
}
void forward(float* input_)override {
input = input_;
input2 = input_;
for (const auto& l : layers) {
l->forward(input);
input = l->get_output();
}
int NN = batch * _c * _h * _w;
residual_forward_kernel << <(NN + 255) / 256, 256 >> > (output, input, input2, NN);
error_handling(cudaGetLastError());
//cudaMemcpy(input2, inputTemp, sizeof(float)*batch * 10, cudaMemcpyDeviceToDevice);
}
void forward2(float* input_)override {
input = input_;
input2 = input_; //batch = 1;
for (const auto& l : layers) {
l->forward2(input);
input = l->get_output();
}
int NN = batch * _c * _h * _w;
//int NN = batch * 32 * 32 * 32;
residual_forward_kernel << <(NN + 255) / 256, 256 >> > (output, input, input2, NN);
// error_handling(cudaGetLastError());
// cudaMemcpy(input2, inputTemp, sizeof(float)*batch * 10, cudaMemcpyDeviceToDevice);
/*const float alpha = 1.0f, beta = 0.0f;
forward(input);*/
}
/*void forward2(float* inputtest)override {
input = inputtest;
const float alpha = 1.0f, beta = 0.0f;
forward(input);
}*/
void backward(float* grad_output)override {//梯度来自残差块后的relu，当前只有一个残差块！！！！！！！！！！！
float* grad = grad_output;//要记住这个梯度，即备份一个
float* grad备用 = grad_output;
for (int i = layers.size() - 1; i >= 0; i--) {
layersi->backward(grad);
grad = layersi->get_grad_input();
}
//float* d_residual = grad备用*X输入数据;//input2 = input_;
// float* d_residual = grad备用*input2;//input2 = input_;
int NN = batch * _c * _h * _w;
/*for (int i = 0; i <NN; i++)
{
d_residuali = grad备用i*input2i;
}*/
int threads = 256;
int blocks = (NN + threads - 1) / threads;
//mulext << <blocks, threads >> >(NN, batch, _c, _h, _w, input2, _c, grad备用);
//// mul << <blocks, threads >> >(grad备用, input2, d_residual, NN);//c为输出=d_residual
//error_handling(cudaGetLastError());
//residual_backprop_kernel << <blocks, threads >> >(grad, grad_input, grad备用, NN);
//error_handling(cudaGetLastError());
//// cudaMemcpy(grad_input, grad, sizeof(float)*batch * 32 * 32 * 32, cudaMemcpyDeviceToDevice);
//使用yolo 的残差试一试，看两个bn有什么情况
mul << <blocks, threads >> > (grad备用, input2, d_residual, NN);//c为输出=d_residual
error_handling(cudaGetLastError());
shortcut_gpu(batch, _w, _h, _c, d_residual, _w, _h, _c, grad);//虚线l.out_c=12,l.c=16,在这里是实线，l.out_c=16,l.c=16
cudaMemcpy(grad_input, grad, sizeof(float) * NN, cudaMemcpyDeviceToDevice);
error_handling(cudaGetLastError());//仍然是第二个bn层方差均值为零
}
int getname() override { return 3; }
float* get_output() override { return output; }
float* get_grad_input() override { return grad_input; }
void update(float lr) {
for (const auto& l : layers) {
l->update(lr);
}
}
~residualExt2() {
cudaFree(output);
cudaFree(grad_input);
}
private:
// cublasHandle_t &cublas;
int _c, _h, _w;
cudnnHandle_t& cudnn;
int batch;
float* input, * output, * grad_input;
float* input2;
float* d_residual;
public:
std::vector<std::shared_ptr<Layer>> layers;
};
上面红色加粗的是新增及修改过的，mul函数修改如下：
global void mul(float* a, float* b, float* c, int N) {
int index = blockIdx.x * blockDim.x + threadIdx.x;
if (index < N) {
//cindex = aindex * (bindex > 0 ? 1 : 0.1);//之前最优版本，leakyrelu放在add之后
cindex = aindex * bindex ;//借鉴darknet，leakyrelu放在add之前了202607101514
// cindex = aindex * (1 - bindex* bindex);//tanhx方式
}
}