知识点回顾
- 规范的文件命名
- 规范的文件夹管理
- 机器学习项目的拆分
- 编码格式和类型注解
**作业：**尝试针对之前的心脏病项目ipynb，将他按照今天的示例项目整理成规范的形式，思考下哪些部分可以未来复用。
心脏病项目目录
python
目录结构:
heart/
├── config/ #集中存放项目的配置文件
├── data/ #存放项目相关数据
├── processed/ #数据预处理后的数据
└── row/ #原始数据
├── experiments/ #用于探索和测试
├── models/ #存放训练好的模型文件
├── reports/ #存储项目运行产生的各类报告和输出文件
└── src/ #存放项目的核心源代码
├── data/ #数据相关代码
├── models/ #模型相关代码
└── utils/ #通用辅助函数代码心脏病项目拆分
导入依赖库
python
# 忽视警告
import warnings
warnings.simplefilter('ignore')
# 数据处理
import numpy as np
import pandas as pd
# 数据可视化
import matplotlib.pyplot as plt
import seaborn as sns
# 随机森林
from sklearn.ensemble import RandomForestClassifier
# 决策树
from sklearn.tree import DecisionTreeClassifier
# 树的可视化
from sklearn.tree import export_graphviz
# 模型评估方法
from sklearn.metrics import roc_curve, auc
from sklearn.metrics import classification_report
# 混淆矩阵
from sklearn.metrics import confusion_matrix
# 数据切分
from sklearn.model_selection import train_test_split
#可解释性分析
import shap
np.random.seed(123)
pd.options.mode.chained_assignment = None
%matplotlib inline数据可视化
python
dt = pd.read_csv("heart.csv")
# 设置可视化风格
sns.set(palette = 'pastel', rc = {"figure.figsize": (10,5), # 图形大小、
"axes.titlesize" : 14, # 标题文字尺寸
"axes.labelsize" : 12, # 坐标轴标签文字尺寸
"xtick.labelsize" : 10, # X轴刻度文字尺寸
"ytick.labelsize" : 10 }) # Y轴刻度文字尺寸
a = sns.countplot(x = 'target', data = dt) # 绘制计数图，其中x为target，数据为dt
a.set_title('Distribution of Presence of Heart Disease') # 设置图形标题
a.set_xticklabels(['Absent', 'Present']) # 将两个条形的标签分别设置为"Absent"（没有心脏病）和"Present"（有心脏病）
plt.xlabel("Presence of Heart Disease") # 设置X轴标签
# 显示图形
plt.show()
#患者年龄分布
plt.show()
g = sns.countplot(x = 'age', data = dt) # 绘制计数图，其中x为age，数据为dt
g.set_title('Distribution of Age') # 设置图形标题
plt.xlabel('Age') # 设置X轴标签
#患者性别分布
dt.sex.value_counts()
b = sns.countplot(x = 'target', data = dt, hue = 'sex') # 创建一个计数图，其中x为target，数据为dt，用sex作为色相（切分类别）
plt.legend(['Female', 'Male']) # 以female/male作为标签，在图形中嵌入图例
b.set_title('Distribution of Presence of Heart Disease by Sex') # 设置图形标题
b.set_xticklabels(['Absent', 'Present']) # 设置条形图的标签
# 显示图形
plt.show()
# 可视化病患血清胆固醇浓度分布
sns.distplot(dt['chol'].dropna(), kde=True, color='darkblue', bins=40)
# 可视化病人(入院时)的静息血压分布
sns.distplot(dt['trestbps'].dropna(), kde=True, color='darkgreen', bins=10)
# 可视化病人空腹血糖浓度分布
g = sns.countplot(x = 'fbs', data = dt) # 绘制计数图，其中x为fbs，数据为dt
g.set_title('Distribution of Fasting blood sugar') # 设置图形标题
plt.xlabel('Fasting blood sugar') # 设置X轴标签
#绘制热力图
f,ax = plt.subplots(figsize=(12,12)) # 定义图形尺寸
# 根据计算的相关值绘制热力图
sns.heatmap(dt.corr('pearson'), annot = True, linewidths = .5, fmt = '.1f', ax = ax)
# 显示特征相关性热力图
plt.show()数据预处理
python
#由于原数据集内的特征名称不利于解读。因此，我们先对其进行重命名，使其更容易理解。
dt.columns = ['age',
'sex',
'chest_pain_type',
'resting_blood_pressure',
'cholesterol',
'fasting_blood_sugar',
'rest_ecg',
'max_heart_rate_achieved',
'exercise_induced_angina',
'st_depression',
'st_slope',
'num_major_vessels',
'thalassemia', 'target']
#转换数据类型
dt.dtypes
# 使用"astype"指定数据类型
dt['sex'] = dt['sex'].astype('object')
dt['chest_pain_type'] = dt['chest_pain_type'].astype('object')
dt['fasting_blood_sugar'] = dt['fasting_blood_sugar'].astype('object')
dt['rest_ecg'] = dt['rest_ecg'].astype('object')
dt['exercise_induced_angina'] = dt['exercise_induced_angina'].astype('object')
dt['st_slope'] = dt['st_slope'].astype('object')
dt['thalassemia'] = dt['thalassemia'].astype('object')
#转换后重新确认输出
dt.dtypes
# 对object数据类型进行编码
# 将"female"编码为0，将"male"编码为1
# 下面的编码方式类似
dt['sex'][dt['sex'] == 0] = 'female'
dt['sex'][dt['sex'] == 1] = 'male'
dt['chest_pain_type'][dt['chest_pain_type'] == 1] = 'typical angina'
dt['chest_pain_type'][dt['chest_pain_type'] == 2] = 'atypical angina'
dt['chest_pain_type'][dt['chest_pain_type'] == 3] = 'non-anginal pain'
dt['chest_pain_type'][dt['chest_pain_type'] == 4] = 'asymptomatic'
dt['fasting_blood_sugar'][dt['fasting_blood_sugar'] == 0] = 'lower than 120mg/ml'
dt['fasting_blood_sugar'][dt['fasting_blood_sugar'] == 1] = 'greater than 120mg/ml'
dt['rest_ecg'][dt['rest_ecg'] == 0] = 'normal'
dt['rest_ecg'][dt['rest_ecg'] == 1] = 'ST-T wave abnormality'
dt['rest_ecg'][dt['rest_ecg'] == 2] = 'left ventricular hypertrophy'
dt['exercise_induced_angina'][dt['exercise_induced_angina'] == 0] = 'no'
dt['exercise_induced_angina'][dt['exercise_induced_angina'] == 1] = 'yes'
dt['st_slope'][dt['st_slope'] == 1] = 'upsloping'
dt['st_slope'][dt['st_slope'] == 2] = 'flat'
dt['st_slope'][dt['st_slope'] == 3] = 'downsloping'
dt['thalassemia'][dt['thalassemia'] == 1] = 'normal'
dt['thalassemia'][dt['thalassemia'] == 2] = 'fixed defect'
dt['thalassemia'][dt['thalassemia'] == 3] = 'reversable defect'
# 调用"get_dummies"进行独特编码
dt = pd.get_dummies(dt, drop_first=True)
#划分训练集和测试集
X_train, X_test, y_train, y_test = train_test_split(dt.drop(columns='target'),
dt['target'],
test_size=0.2,
random_state=10)模型创建和预测
python
#创建模型
model = RandomForestClassifier(max_depth=5, n_estimators=10) # 设置最大深度与基学习器等参数
model.fit(X_train, y_train) # 使用随机森林拟合训练集
#模型预测
y_predict = model.predict(X_test)
# 生成一个nxm的矩阵，第i行表示第i个样本属于各个标签的概率
y_pred_quant = model.predict_proba(X_test)[:, 1]
y_pred_bin = model.predict(X_test)模型评估
python
#生成混淆矩阵
confusion_matrix = confusion_matrix(y_test, y_pred_bin)
confusion_matrix
#计算灵敏度和特异度
total=sum(sum(confusion_matrix))
sensitivity = confusion_matrix[0,0]/(confusion_matrix[0,0]+confusion_matrix[1,0])
print('灵敏度 : ', sensitivity )
specificity = confusion_matrix[1,1]/(confusion_matrix[1,1]+confusion_matrix[0,1])
print('特异度 : ', specificity)
# 绘制ROC曲线
# 得到曲线的横轴和纵轴
fpr, tpr, thresholds = roc_curve(y_test, y_pred_quant)
fig, ax = plt.subplots()
# 绘制roc曲线
ax.plot(fpr, tpr)
# 绘制y=x直线
ax.plot([0, 1], [0, 1], transform=ax.transAxes, ls="--", c=".3")
plt.xlim([0.0, 1.0])
plt.ylim([0.0, 1.0])
plt.rcParams['font.size'] = 12
plt.title('ROC curve for diabetes classifier')
plt.xlabel('False Positive Rate (1 - Specificity)')
plt.ylabel('True Positive Rate (Sensitivity)')
plt.grid(True)可解释性分析
python
explainer = shap.TreeExplainer(model)
shap_values = explainer.shap_values(X_test)
#特征重要性图
shap.summary_plot(shap_values[1], X_test, plot_type="bar")
#蜂群图
shap.summary_plot(shap_values[1], X_test)