R语言实现对模型的参数优化与评价KS曲线、ROC曲线、深度学习模型训练、交叉验证、网格搜索

目录

一、模型性能评估

1、数据预测评估

2、概率预测评估

二、模型参数优化

1、训练集、验证集、测试集的引入

2、k折线交叉验证

2、网格搜索

---

一、模型性能评估

1 、数据预测评估

### 数据预测评估 ###

# 加载包，不存在就进行在线下载后加载

if(!require(mlbench)) install.packages("mlbench")

library(mlbench)

data("BostonHousing")

# 数据分区

library(caret)

library(ggplot2)

library(lattice)

index <- createDataPartition(BostonHousing$medv,p = 0.75,list = FALSE)

train <- BostonHousing[index,]

test <- BostonHousing[-index,]

# 利用训练集构建模型,并对测试集进行预测

set.seed(1234)

fit <- lm(medv ~ .,data = train)

pred <- predict(fit,newdata = test)

# 自定义函数计算数值预测模型的评估指标

numericIndex <- function(obs,pred){

  # 计算平均绝对误差MAE

  MAE <- mean(abs(obs-pred))

  # 计算均方误差MSE

  MSE <- mean((obs-pred)^2)

  # 计算均方根误差RMSE

  RMSE <- sqrt(mean((obs-pred)^2))

  # 计算归一化均方误差

  NMSE <- sum((obs-pred)^2)/(sum((obs-mean(obs))^2))

  # 计算判定系数Rsquared

  Rsqured <- cor(pred,obs)^2

  # 返回向量形式

  return(c('MAE' = MAE,'MSE' = MSE,'RMSE' = RMSE,'NMSE' = NMSE,'Rsqured' = Rsqured))

}

# 计算各指标度量值

numericIndex(test$medv,pred)

# 利用caret包

library(caret)

postResample(pred,test$medv)

2 、概率预测评估

### 混淆矩阵 ###

# install.packages("DAAG")

library(DAAG)

data(anesthetic)

anes1=glm(factor(nomove)~conc,family=binomial(link='logit'),data=anesthetic)

# 对模型做出预测结果

pre=predict(anes1,type='response') # 得到的是样本为1类别时的预测概率值

# 以0.5作为分界点

result <- ifelse(pre>0.5,1,0)

# 构建混淆矩阵

confusion<-table(actual=anesthetic$nomove,predict=result)

confusion

# 计算各指标（1为正样本，0为负样本）

(TP <- confusion[4])

(TN <- confusion[1])

(FP <- confusion[3])

(FN <- confusion[2])

(Accuracy <- (sum(TN) + sum(TP))/sum(confusion)) #准确率

(Accuracy <- (TN + TP)/sum(confusion)) #准确率

(Precision <- TP/(TP+FP)) # 精度

(Recall <- TP/(TP+FN)) # 灵敏性/召回率

(F1 <- 2*TP/(2*TP+FP+FN)) # F1-score

(FPR <- FP/(TN+FP)) #假正率

# 使用confusionMatrix函数

library(caret)

confusionMatrix(data = factor(result), # 预测结果

                reference = factor(anesthetic$nomove), # 实际结果

                positive = '1', # 指定类别1为正样本

                mode = "prec_recall") # 设置为精度和查全率模式

### ROC曲线  ###

# 构建结果数据集

result <- data.frame(pre_prob = pre,true_label = anesthetic$nomove)

result <- result[order(result$pre_prob,decreasing = T),] # 按照预测概率值进行降序排序

result$cumsum <-  cumsum(rep(1,nrow(result))) # 统计累计样本数量

result$poscumsum <- cumsum(result$true_label) # 统计累计正样本数量

result$tpr <- round(result$poscumsum/sum(result$true_label==1),3) # 计算真正率

result$fpr <- round((result$cumsum-result$poscumsum)/sum(result$true_label==0),3) # 计算假正率

result$lift <- round((result$poscumsum/result$cumsum)/(sum(result$true_label==1)/nrow(result)),2) # 计算提升度

head(result)

tail(result)

# 画出roc曲线

library(ggplot2)

if(!require(ROCR)) install.packages("ROCR")

library(ROCR)

ggplot(result) +

  geom_line(aes(x = result$fpr, y = result$tpr),color = "red1",size = 1.2) +

  geom_segment(aes(x = 0, y = 0, xend = 1, yend = 1), color = "grey", lty = 2,size = 1.2) +

  annotate("text", x = 0.5, y = 1.05,

           label=paste('AUC:',round(ROCR::performance(prediction(result$pre_prob, result$true_label),'auc')@y.values[[1]],3)),

           size=6, alpha=0.8) +

  scale_x_continuous(breaks=seq(0,1,.2))+

  scale_y_continuous(breaks=seq(0,1,.2))+

  xlab("False Postive Rate")+

  ylab("True Postive Rate")+

  ggtitle(label="ROC - Chart")+

  theme_bw()+

  theme(

    plot.title=element_text(colour="gray24",size=12,face="bold"),

    plot.background = element_rect(fill = "gray90"),

    axis.title=element_text(size=10),

    axis.text=element_text(colour="gray35"))

# 利用ROCR包绘制roc曲线

library(ROCR)

pred1 <- prediction(pre,anesthetic$nomove)

# 设置参数，横轴为假正率fpr，纵轴为真正率tpr

perf <- performance(pred1,'tpr','fpr')

# 绘制ROC曲线

plot(perf,main = "利用ROCR包绘制ROC曲线")

# 计算AUC值

auc.adj <- performance(pred1,'auc')

auc <- auc.adj@y.values[[1]]

auc

# 画出KS曲线

ggplot(result) +

  geom_line(aes((1:nrow(result))/nrow(result),result$tpr),colour = "red2",size = 1.2) +

  geom_line(aes((1:nrow(result))/nrow(result),result$fpr),colour = "blue3",size = 1.2) +

  annotate("text", x = 0.5, y = 1.05, label=paste("KS=", round(which.max(result$tpr-result$fpr)/nrow(result), 4),

                                                  "at Pop=", round(max(result$tpr-result$fpr), 4)), size=6, alpha=0.8)+

  scale_x_continuous(breaks=seq(0,1,.2))+

  scale_y_continuous(breaks=seq(0,1,.2))+

  xlab("Total Population Rate")+

  ylab("TP/FP Rate")+

  ggtitle(label="KS - Chart")+

  theme_bw()+

  theme(

    plot.title=element_text(colour="gray24",size=12,face="bold"),

    plot.background = element_rect(fill = "gray90"),

    axis.title=element_text(size=10),

    axis.text=element_text(colour="gray35"))

# 画累积提升图

ggplot(result) +

  geom_line(aes(x = (1:nrow(result))/nrow(result), y = result$lift),color = "red3",size = 1.2) +

  scale_x_continuous(breaks=seq(0,1,.2))+

  xlab("Total Population Rate")+

  ylab("Lift value")+

  ggtitle(label="LIFT - Chart")+

  theme_bw()+

  theme(

    plot.title=element_text(colour="gray24",size=12,face="bold"),

    plot.background = element_rect(fill = "gray90"),

    axis.title=element_text(size=10),

    axis.text=element_text(colour="gray35"))

# 读入封装好的R代码

source('自定义绘制各种曲线函数.R')

# 加载ROCR.simple数据集

library(ROCR)

data(ROCR.simple)

# 绘制各种曲线

pc <- plotCurve(pre_prob=ROCR.simple$predictions,

                true_label=ROCR.simple$labels)

# 查看各种曲线

library(gridExtra)

grid.arrange(pc$roc_curve,pc$ks_curve,pc$lift_curve,ncol = 3)

二、模型参数优化

1、训练集、验证集、测试集的引入

###   训练集、验证集、测试集的引入  ###

#注意：以下代码需要安装tensorflow和keras包才能运行

devtools::install_github("rstudio/tensorflow")

library(tensorflow)

install_tensorflow()

library(keras)

# 导入数据集

library(keras)

c(c(x_train,y_train),c(x_test,y_test )) %<-% dataset_mnist()

# 查看数据集的维度

cat('x_train shape:',dim(x_train))

cat('y_train shape:',dim(y_train))

cat('x_test shape:',dim(x_test))

cat('y_test shape:',dim(y_test))

# 对数字图像进行可视化

par(mfrow=c(3,3))

for(i in 1:9){

  plot(as.raster(x_train[i,,],max = 255))

  title(main = paste0('数字标签为：',y_train[i]))

}

par(mfrow = c(1,1))


# 数据预处理

x_train <- array_reshape(x_train,c(nrow(x_train),784))

x_test <- array_reshape(x_test,c(nrow(x_test),784))

x_train <- x_train / 255

x_test <- x_test / 255

y_train <- to_categorical(y_train,10)

y_test <- to_categorical(y_test,10)



# 构建网络结构

model <- keras_model_sequential()

model %>%

  layer_dense(units = 256,activation = 'relu',input_shape = c(784)) %>%

  layer_dense(units = 128,activation = 'relu') %>%

  layer_dense(units = 10,activation = 'softmax')

summary(model)

> # 编译和训练深度学习模型

> model %>%

+   compile(loss = 'categorical_crossentropy',

+           optimizer = optimizer_rmsprop(),

+           metrics = c('accuracy'))

> history <- model %>% fit(

+   x_train,y_train,

+   epochs = 10,batch_size = 128,

+   validation_split = 0.2

+ )

Epoch 1/10

1/375 ━━━━━━━━━━━━━━━━━━━━ 2:25 389ms/step - accuracy: 0.0547 - loss: 2.3528

19/375 ━━━━━━━━━━━━━━━━━━━━ 1s 3ms/step - accuracy: 0.5331 - loss: 1.5280

39/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.6426 - loss: 1.2044

60/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.6974 - loss: 1.0292

80/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.7294 - loss: 0.9236

99/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.7511 - loss: 0.8515

119/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.7683 - loss: 0.7934

140/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.7827 - loss: 0.7446

160/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.7938 - loss: 0.7066

179/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.8028 - loss: 0.6759

201/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.8117 - loss: 0.6454

220/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.8185 - loss: 0.6224

240/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.8247 - loss: 0.6009

261/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.8305 - loss: 0.5809

282/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.8357 - loss: 0.5630

303/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.8404 - loss: 0.5468

323/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.8445 - loss: 0.5327

344/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.8484 - loss: 0.5191

363/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.8517 - loss: 0.5077

375/375 ━━━━━━━━━━━━━━━━━━━━ 2s 4ms/step - accuracy: 0.8538 - loss: 0.5004 - val_accuracy: 0.9590 - val_loss: 0.1390

Epoch 2/10

1/375 ━━━━━━━━━━━━━━━━━━━━ 8s 22ms/step - accuracy: 0.9688 - loss: 0.1577

19/375 ━━━━━━━━━━━━━━━━━━━━ 1s 3ms/step - accuracy: 0.9593 - loss: 0.1446

37/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9586 - loss: 0.1431

55/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9581 - loss: 0.1421

72/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9581 - loss: 0.1414

92/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9581 - loss: 0.1412

111/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9580 - loss: 0.1407

130/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9582 - loss: 0.1397

150/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9585 - loss: 0.1387

171/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9587 - loss: 0.1377

191/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9589 - loss: 0.1367

211/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9592 - loss: 0.1358

230/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9594 - loss: 0.1349

250/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9596 - loss: 0.1340

269/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9598 - loss: 0.1332

291/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9601 - loss: 0.1322

311/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9603 - loss: 0.1314

331/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9605 - loss: 0.1307

352/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9607 - loss: 0.1300

372/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9609 - loss: 0.1293

375/375 ━━━━━━━━━━━━━━━━━━━━ 1s 3ms/step - accuracy: 0.9610 - loss: 0.1292 - val_accuracy: 0.9680 - val_loss: 0.1072

Epoch 3/10

1/375 ━━━━━━━━━━━━━━━━━━━━ 8s 23ms/step - accuracy: 0.9453 - loss: 0.1397

21/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9727 - loss: 0.0838

41/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9750 - loss: 0.0806

59/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9759 - loss: 0.0788

78/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9763 - loss: 0.0776

99/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9764 - loss: 0.0771

119/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9765 - loss: 0.0770

139/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9764 - loss: 0.0773

161/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9764 - loss: 0.0776

183/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9763 - loss: 0.0778

205/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9763 - loss: 0.0778

224/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9763 - loss: 0.0778

244/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9764 - loss: 0.0777

264/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9764 - loss: 0.0777

282/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9764 - loss: 0.0776

301/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9765 - loss: 0.0775

319/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9765 - loss: 0.0774

337/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9765 - loss: 0.0773

356/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9765 - loss: 0.0773

375/375 ━━━━━━━━━━━━━━━━━━━━ 1s 3ms/step - accuracy: 0.9766 - loss: 0.0772 - val_accuracy: 0.9735 - val_loss: 0.0908

Epoch 4/10

1/375 ━━━━━━━━━━━━━━━━━━━━ 8s 24ms/step - accuracy: 0.9766 - loss: 0.0345

22/375 ━━━━━━━━━━━━━━━━━━━━ 0s 2ms/step - accuracy: 0.9827 - loss: 0.0557

42/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9834 - loss: 0.0553

63/375 ━━━━━━━━━━━━━━━━━━━━ 0s 2ms/step - accuracy: 0.9832 - loss: 0.0555

85/375 ━━━━━━━━━━━━━━━━━━━━ 0s 2ms/step - accuracy: 0.9830 - loss: 0.0560

105/375 ━━━━━━━━━━━━━━━━━━━━ 0s 2ms/step - accuracy: 0.9830 - loss: 0.0561

125/375 ━━━━━━━━━━━━━━━━━━━━ 0s 2ms/step - accuracy: 0.9830 - loss: 0.0561

146/375 ━━━━━━━━━━━━━━━━━━━━ 0s 2ms/step - accuracy: 0.9830 - loss: 0.0562

167/375 ━━━━━━━━━━━━━━━━━━━━ 0s 2ms/step - accuracy: 0.9829 - loss: 0.0563

186/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9829 - loss: 0.0564

204/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9829 - loss: 0.0564

221/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9828 - loss: 0.0565

241/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9828 - loss: 0.0565

261/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9828 - loss: 0.0565

281/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9828 - loss: 0.0564

301/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9828 - loss: 0.0564

320/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9828 - loss: 0.0563

339/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9828 - loss: 0.0562

357/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9828 - loss: 0.0562

375/375 ━━━━━━━━━━━━━━━━━━━━ 1s 3ms/step - accuracy: 0.9828 - loss: 0.0562 - val_accuracy: 0.9747 - val_loss: 0.0845

Epoch 5/10

1/375 ━━━━━━━━━━━━━━━━━━━━ 7s 21ms/step - accuracy: 1.0000 - loss: 0.0048

21/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9920 - loss: 0.0268

41/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9910 - loss: 0.0300

62/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9907 - loss: 0.0303

82/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9904 - loss: 0.0309

102/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9900 - loss: 0.0317

122/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9897 - loss: 0.0325

142/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9895 - loss: 0.0333

163/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9893 - loss: 0.0339

183/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9892 - loss: 0.0344

203/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9890 - loss: 0.0350

223/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9889 - loss: 0.0354

244/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9888 - loss: 0.0359

262/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9887 - loss: 0.0362

280/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9886 - loss: 0.0366

300/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9885 - loss: 0.0369

321/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9884 - loss: 0.0372

341/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9883 - loss: 0.0375

360/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9883 - loss: 0.0377

375/375 ━━━━━━━━━━━━━━━━━━━━ 1s 3ms/step - accuracy: 0.9882 - loss: 0.0379 - val_accuracy: 0.9728 - val_loss: 0.0921

Epoch 6/10

1/375 ━━━━━━━━━━━━━━━━━━━━ 9s 25ms/step - accuracy: 1.0000 - loss: 0.0120

20/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9924 - loss: 0.0235

39/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9915 - loss: 0.0258

58/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9911 - loss: 0.0267

78/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9910 - loss: 0.0270

99/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9908 - loss: 0.0273

118/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9907 - loss: 0.0277

138/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9907 - loss: 0.0280

157/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9906 - loss: 0.0284

175/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9905 - loss: 0.0288

194/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9904 - loss: 0.0291

213/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9904 - loss: 0.0294

233/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9903 - loss: 0.0296

254/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9903 - loss: 0.0298

275/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9903 - loss: 0.0300

296/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9903 - loss: 0.0302

317/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9902 - loss: 0.0303

337/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9902 - loss: 0.0305

358/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9902 - loss: 0.0306

375/375 ━━━━━━━━━━━━━━━━━━━━ 1s 3ms/step - accuracy: 0.9902 - loss: 0.0307 - val_accuracy: 0.9768 - val_loss: 0.0857

Epoch 7/10

1/375 ━━━━━━━━━━━━━━━━━━━━ 9s 25ms/step - accuracy: 1.0000 - loss: 0.0091

20/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9955 - loss: 0.0147

39/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9948 - loss: 0.0171

58/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9946 - loss: 0.0183

77/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9945 - loss: 0.0192

95/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9944 - loss: 0.0196

114/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9944 - loss: 0.0197

133/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9943 - loss: 0.0199

154/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9943 - loss: 0.0201

175/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9941 - loss: 0.0203

195/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9940 - loss: 0.0206

216/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9939 - loss: 0.0208

237/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9938 - loss: 0.0211

258/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9937 - loss: 0.0213

278/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9936 - loss: 0.0215

299/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9935 - loss: 0.0218

319/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9934 - loss: 0.0220

339/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9933 - loss: 0.0222

359/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9933 - loss: 0.0223

375/375 ━━━━━━━━━━━━━━━━━━━━ 1s 3ms/step - accuracy: 0.9932 - loss: 0.0225 - val_accuracy: 0.9763 - val_loss: 0.0927

Epoch 8/10

1/375 ━━━━━━━━━━━━━━━━━━━━ 8s 22ms/step - accuracy: 1.0000 - loss: 0.0030

21/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9955 - loss: 0.0162

42/375 ━━━━━━━━━━━━━━━━━━━━ 0s 2ms/step - accuracy: 0.9952 - loss: 0.0177

62/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9950 - loss: 0.0180

83/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9950 - loss: 0.0181

104/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9950 - loss: 0.0179

125/375 ━━━━━━━━━━━━━━━━━━━━ 0s 2ms/step - accuracy: 0.9949 - loss: 0.0180

147/375 ━━━━━━━━━━━━━━━━━━━━ 0s 2ms/step - accuracy: 0.9948 - loss: 0.0181

168/375 ━━━━━━━━━━━━━━━━━━━━ 0s 2ms/step - accuracy: 0.9947 - loss: 0.0181

188/375 ━━━━━━━━━━━━━━━━━━━━ 0s 2ms/step - accuracy: 0.9946 - loss: 0.0181

209/375 ━━━━━━━━━━━━━━━━━━━━ 0s 2ms/step - accuracy: 0.9945 - loss: 0.0181

229/375 ━━━━━━━━━━━━━━━━━━━━ 0s 2ms/step - accuracy: 0.9945 - loss: 0.0182

247/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9945 - loss: 0.0182

265/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9944 - loss: 0.0182

284/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9944 - loss: 0.0182

303/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9944 - loss: 0.0183

322/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9944 - loss: 0.0183

341/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9943 - loss: 0.0183

358/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9943 - loss: 0.0184

375/375 ━━━━━━━━━━━━━━━━━━━━ 1s 3ms/step - accuracy: 0.9943 - loss: 0.0184 - val_accuracy: 0.9790 - val_loss: 0.0842

Epoch 9/10

1/375 ━━━━━━━━━━━━━━━━━━━━ 8s 24ms/step - accuracy: 1.0000 - loss: 0.0019

20/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9972 - loss: 0.0090

40/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9971 - loss: 0.0098

60/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9970 - loss: 0.0100

79/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9970 - loss: 0.0102

100/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9969 - loss: 0.0103

120/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9968 - loss: 0.0106

140/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9968 - loss: 0.0108

161/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9967 - loss: 0.0110

181/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9967 - loss: 0.0111

201/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9967 - loss: 0.0113

222/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9967 - loss: 0.0114

242/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9966 - loss: 0.0116

260/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9965 - loss: 0.0117

277/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9965 - loss: 0.0118

298/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9964 - loss: 0.0119

319/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9964 - loss: 0.0121

340/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9963 - loss: 0.0122

360/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9963 - loss: 0.0124

375/375 ━━━━━━━━━━━━━━━━━━━━ 1s 3ms/step - accuracy: 0.9962 - loss: 0.0125 - val_accuracy: 0.9783 - val_loss: 0.0885

Epoch 10/10

1/375 ━━━━━━━━━━━━━━━━━━━━ 30s 82ms/step - accuracy: 1.0000 - loss: 0.0014

20/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9981 - loss: 0.0071

40/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9973 - loss: 0.0084

59/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9970 - loss: 0.0088

78/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9970 - loss: 0.0090

98/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9969 - loss: 0.0093

118/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9969 - loss: 0.0094

137/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9969 - loss: 0.0096

156/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9968 - loss: 0.0098

176/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9967 - loss: 0.0100

195/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9967 - loss: 0.0101

215/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9966 - loss: 0.0102

236/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9966 - loss: 0.0103

256/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9966 - loss: 0.0105

276/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9965 - loss: 0.0106

296/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9965 - loss: 0.0106

316/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9965 - loss: 0.0107

335/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9965 - loss: 0.0107

354/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9965 - loss: 0.0107

374/375 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step - accuracy: 0.9965 - loss: 0.0108

375/375 ━━━━━━━━━━━━━━━━━━━━ 1s 3ms/step - accuracy: 0.9965 - loss: 0.0108 - val_accuracy: 0.9798 - val_loss: 0.0888

plot(history)

# 评估模型效果

DNN_score <- model %>% evaluate(x_test,y_test)

DNN_score$acc # 查看测试集的准确率

2 、k折线交叉验证

### 10折交叉验证 ###

# 导入car数据集

car <- read.table("../data/car.data",sep = ",")

# 对变量重命名

colnames(car) <- c("buy","main","doors","capacity",

                   "lug_boot","safety","accept")



# 手动构建10折交叉验证

#下面构造10折下标集

library(caret)

ind<-createFolds(car$accept,k=10,list=FALSE,returnTrain=FALSE)

# 下面再做10折交叉验证，这里仅给出训练集和测试集的分类平均误判率。

E0=rep(0,10);E1=E0

car$accept<-as.factor(car$accept)

library(C50)

for(i in 1:10){

  n0=nrow(car)-nrow(car[ind==i,]);n1=nrow(car[ind==i,])

  a=C5.0(accept~.,car[!ind==i,])

  E0[i]=sum(car[!ind==i,'accept']!=predict(a,car[!ind==i,]))/n0

  E1[i]=sum(car[ind==i,'accept']!=predict(a,car[ind==i,]))/n1

}

(1-mean(E0));(1-mean(E1))

# 利用caret包中的trainControl函数完成交叉验证

library(caret)

library(ROCR)

control <- trainControl(method="repeatedcv",number=10,repeats=3)

model <- train(accept~.,data=car,method="rpart",

               trControl=control)

model

plot(model)

2 、网格搜索

### 网格搜索 ###

### 网格搜索 ###

#install.packages("gbm")

set.seed(1234)

library(caret)

library(gbm)fitControl <- trainControl(method = 'repeatedcv',

                           number = 10,

                           repeats = 5)

# 设置网格搜索的参数池

gbmGrid <- expand.grid(interaction.depth = c(3,5,9),

                       n.trees = (1:20)*5,

                       shrinkage = 0.1,

                       n.minobsinnode = 20)

nrow(gbmGrid)

# 训练模型，找出最优参数组合

gbmfit <- train(accept ~ .,data = car,

                method = 'gbm',

                trControl = fitControl,

                tuneGrid = gbmGrid,

                metric = 'Accuracy')



gbmfit$bestTune # 查看模型最优的参数组合

plot(gbmfit)