- 🍨 本文为🔗365天深度学习训练营 中的学习记录博客
- 🍖 原作者：K同学啊
一、环境及数据准备
1. 我的环境
- 语言环境：Python3.11.9
- 编译器：Jupyter notebook
- 深度学习框架：TensorFlow 2.15.0
2. 导入数据
python
import numpy as np
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
import warnings
warnings.filterwarnings('ignore')
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import MinMaxScaler
import tensorflow as tf
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, Activation,Dropout
from tensorflow.keras.callbacks import EarlyStopping
from tensorflow.keras.layers import Dropout
from sklearn.metrics import classification_report,confusion_matrix
from sklearn.metrics import r2_score
from sklearn.metrics import mean_absolute_error , mean_absolute_percentage_error , mean_squared_error
python
data = pd.read_csv(r"D:\Personal Data\Learning Data\DL Learning Data\weatherAUS.csv")
df = data.copy()
data.head()输出：
python
data.dtypes
python
data['Date'] = pd.to_datetime(data['Date'])
data['Date']输出：
python
data['year'] = data['Date'].dt.year
data['Month'] = data['Date'].dt.month
data['day'] = data['Date'].dt.day
data.head()输出：
python
data.drop('Date', axis=1, inplace=True)
python
data.columns输出：
二、探索式数据分析
1. 数据相关性探索
python
plt.figure(figsize=(15,13))
numeric_data = data.select_dtypes(include=[np.number])
# data.corr()表示了data中的两个变量之间的相关性
ax = sns.heatmap(numeric_data.corr(), square=True, annot=True, fmt='.2f')
ax.set_xticklabels(ax.get_xticklabels(), rotation=90)
plt.show()输出：
2.是否会下雨
python
sns.set(style="darkgrid")
plt.figure(figsize=(4,3))
sns.countplot(x='RainTomorrow',data=data)输出：
python
plt.figure(figsize=(4,3))
sns.countplot(x='RainToday',data=data)
python
x = pd.crosstab(data['RainTomorrow'], data['RainToday'])
x输出：
python
y = x / x.transpose().sum().values.reshape(2,1)*100
y- 如果今天不下雨，那么明天下雨的机会 = 53.22%
- 如果今天下雨明天下雨的机会 = 46.78%
python
y.plot(kind='bar', figsize=(4, 3), color=['#006666', '#d279a6'])
3. 地理位置与下雨的关系
python
x=pd.crosstab(data['Location'],data['RainToday'])
#获取每个城市下雨天数和非下雨天数的百分比
y=x/x.transpose().sum().values.reshape((-1,1))*100
#按每个城市的雨天百分比排序
y=y.sort_values(by='Yes',ascending=True)
color=['#cc6699','#006699','#006666','#862d86','#ff9966' ]
y.Yes.plot(kind="barh",figsize=(15,20),color=color)输出：
4.湿度和压力对下雨的影响
python
plt.figure(figsize=(8,6))
sns.scatterplot(data=data,x='Pressure9am',y='Pressure3pm',hue='RainTomorrow')
python
plt.figure(figsize=(8,6))
sns.scatterplot(data=data,x='Humidity9am',y='Humidity3pm',hue='RainTomorrow')输出：
低压与高湿度会增加第二天下雨的概率，尤其下午3点的空气湿度。
5. 气温对下雨的影响
python
plt.figure(figsize=(8,6))
sns.scatterplot(x='MaxTemp',y='MinTemp',data=data,hue='RainTomorrow')
结论：当一天的最高气温和最低气温接近时，第二天下雨的概率会增加。
三、数据预处理
1. 处理缺损值
python
#每列中缺失数据的百分比
data.isnull().sum()/data.shape[0]*100输出：
python
#在该列中随机选择数进行填充
lst=['Evaporation','Sunshine','Cloud9am','Cloud3pm']
for col in lst:
fill_list =data[col].dropna()
data[col] =data[col].fillna(pd.Series(np.random.choice(fill_list,size=len(data.index))))
s=(data.dtypes =="object")
object_cols=list(s[s].index)
object_cols输出：
python
#inplace=True：直接修改原对象，不创建副本
#data[i].mode()[0] 返回频率出现最高的选项，众数
for i in object_cols:
data[i].fillna(data[i].mode()[0],inplace=True)
t=(data.dtypes =="float64")
num_cols=list(t[t].index)
num_cols输出：
python
#.median(), 中位数
for i in num_cols:
data[i].fillna(data[i].median(), inplace=True)
data.isnull().sum()输出：
2. 构建数据集
python
from sklearn.preprocessing import LabelEncoder
label_encoder=LabelEncoder()
for i in object_cols:
data[i] =label_encoder.fit_transform(data[i])
X=data.drop(['RainTomorrow','day'],axis=1).values
y=data['RainTomorrow'].values
X_train,X_test, y_train, y_test =train_test_split(X,y,test_size=0.25,random_state=101)
scaler=MinMaxScaler()
scaler.fit(X_train)
X_train=scaler.transform(X_train)
X_test =scaler.transform(X_test)
python
model=Sequential()
model.add(Dense(units=24,activation='tanh',))
model.add(Dense(units=18,activation='tanh'))
model.add(Dense(units=23,activation='tanh'))
model.add(Dropout(0.5))
model.add(Dense(units=12,activation='tanh'))
model.add(Dropout(0.2))
model.add(Dense(units=1,activation='sigmoid'))四、 预测是否会下雨
1. 搭建神经网络
python
from tensorflow.keras.optimizers import Adam
optimizer=tf.keras.optimizers.Adam(learning_rate=1e-4)
model.compile(loss='binary_crossentropy',
optimizer=optimizer,
metrics="accuracy")
python
early_stop=EarlyStopping(monitor='val_loss',
mode='min',
min_delta=0.001,
verbose=1,
patience=25,
restore_best_weights=True)2. 模型训练
python
history=model.fit(x=X_train,
y=y_train,
validation_data=(X_test,y_test), verbose=1,
callbacks=[early_stop],
epochs =10,
batch_size =32
)
3. 结果可视化
python
import matplotlib.pyplot as plt
acc = model.history.history['accuracy']
val_acc = model.history.history['val_accuracy']
loss = model.history.history['loss']
val_loss = model.history.history['val_loss']
epochs_range = range(10)
plt.figure(figsize=(14, 4))
plt.subplot(1, 2, 1)
plt.plot(epochs_range, acc, label='Training Accuracy')
plt.plot(epochs_range, val_acc, label='Validation Accuracy')
plt.legend(loc='lower right')
plt.title('Training and Validation Accuracy')
plt.subplot(1, 2, 2)
plt.plot(epochs_range, loss, label='Training Loss')
plt.plot(epochs_range, val_loss, label='Validation Loss')
plt.legend(loc='upper right')
plt.title('Training and Validation Loss')
plt.show()输出：
五、总结
- 数据预处理中，数据缺损严重时，可在该列中选择数进行填充
- 数据相关性研究可帮助参数的调节