机器学习随机森林

random_forest

随机森林（ Random Forest ） 是一种集成学习方法：用多棵决策树（本实现基于 CART/ClassificationTree）投票，降低单棵树的过拟合与方差。

核心思想

单棵决策树容易过拟合、不稳定。随机森林通过 " 双随机 + 多数投票 " 训练多棵差异化的树，再综合结果：

1. 行抽样（ Bootstrap ）：每棵树用有放回抽样得到的子集训练
2. 列抽样（特征随机）：每棵树只用部分特征建树
3. 汇总：分类时多数投票（bincount(...).argmax()）

树与树不完全相同 → 错误不易同时犯 → 整体更稳、更准。

训练流程（对应代码）

步骤	含义	notebook
建森林	创建 n_estimators 棵空树	init
① 行随机	有放回抽 个样本	bootstrap_sampling：choice(..., replace=True)
② 列随机	再随机选 max_features 个特征（默认约）	fit 里 choice(n_features, max_features)
③ 学树	在该子集上拟合 CART 分类树	self.treesi.fit(sub_X, sub_y)
④ 记特征下标	预测时用同一批特征列	feature_indices

预测流程

对每个测试样本：

• 每棵树用各自保存的特征列做预测
• 把所有树的类别票数统计起来，取票数最多的类

为什么有效

机制	作用
Bootstrap	每棵树看到略有不同的数据
特征随机	进一步增加树的多样性，削弱强特征垄断
投票/平均	降低方差，抑制过拟合

相对单棵树：偏差通常相近，方差更小，泛化更好。

常用参数（本实现）

• n_estimators：树的数量
• max_features：每棵树可用特征数
• max_depth / min_samples_split / min_gain：控制单棵树复杂度

一句话： 随机森林 = 多棵 CART 树 + Bootstrap 行抽样 + 特征列抽样 + 多数投票。

** **代码** **

import numpy as np
# 该模块为自定义模块，封装了构建决策树的基本方法
from cart import *
from sklearn.datasets import make_classification
from sklearn.model_selection import train_test_split
# 树的棵数
n_estimators = 10
# 列抽样最大特征数
max_features = 15
# 生成模拟二分类数据集
X, y = make_classification(n_samples=1000, n_features=20, n_redundant=0, n_informative=2,
                           random_state=1, n_clusters_per_class=1)
rng = np.random.RandomState(2)
X += 2 * rng.uniform(size=X.shape)
# 划分数据集
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3)
print(X_train.shape, y_train.shape, X_test.shape, y_test.shape)

** **输出** **

(700, 20) (700,) (300, 20) (300,)

** **代码** **

# 合并训练数据和标签
X_y = np.concatenate([X, y.reshape(-1,1)], axis=1)
np.random.shuffle(X_y)
m = X_y.shape[0]
sampling_subsets = []

for _ in range(n_estimators):
    idx = np.random.choice(m, m, replace=True)
    bootstrap_Xy = X_y[idx, :]
    bootstrap_X = bootstrap_Xy[:, :-1]
    bootstrap_y = bootstrap_Xy[:, -1]
    sampling_subsets.append([bootstrap_X, bootstrap_y])

** **代码** **

sampling_subsets[0][0].shape

** **输出** **

(1000, 20)

代码

# 自助抽样选择训练数据子集
def bootstrap_sampling(X, y):
    X_y = np.concatenate([X, y.reshape(-1,1)], axis=1)
    np.random.shuffle(X_y)
    n_samples = X.shape[0]
    sampling_subsets = []

    for _ in range(n_estimators):
        # 第一个随机性，行抽样
        idx1 = np.random.choice(n_samples, n_samples, replace=True)
        bootstrap_Xy = X_y[idx1, :]
        bootstrap_X = bootstrap_Xy[:, :-1]
        bootstrap_y = bootstrap_Xy[:, -1]
        sampling_subsets.append([bootstrap_X, bootstrap_y])
    return sampling_subsets

** **代码** **

sampling_subsets = bootstrap_sampling(X_train, y_train)
sub_X, sub_y = sampling_subsets[0]
print(sub_X.shape, sub_y.shape)

** **输出** **

(700, 20) (700,)

** **代码** **

trees = []
# 基于决策树构建森林
for _ in range(n_estimators):
    tree = ClassificationTree(min_samples_split=2, min_gini_impurity=999,
                              max_depth=3)
    trees.append(tree)

trees[0]

** **输出** **

<cart.ClassificationTree at 0x1d5e15aaa20>

** **代码** **

# 随机森林训练
def fit(X, y):
    # 对森林中每棵树训练一个双随机抽样子集
    n_features = X.shape[1]
    sub_sets = bootstrap_sampling(X, y)
    for i in range(n_estimators):
        sub_X, sub_y = sub_sets[i]
        # 第二个随机性，列抽样
        idx2 = np.random.choice(n_features, max_features, replace=True)
        sub_X = sub_X[:, idx2]
        trees[i].fit(sub_X, sub_y)
        trees[i].feature_indices = idx2
        print('The {}th tree is trained done...'.format(i+1))

fit(X_train, y_train)

** **输出** **

The 1th tree is trained done...

The 2th tree is trained done...

The 3th tree is trained done...

The 4th tree is trained done...

The 5th tree is trained done...

The 6th tree is trained done...

The 7th tree is trained done...

The 8th tree is trained done...

The 9th tree is trained done...

The 10th tree is trained done...

** **代码** **

y_preds = []
for i in range(n_estimators):
    idx = trees[i].feature_indices
    sub_X = X_test[:, idx]
    y_pred = trees[i].predict(sub_X)
    y_preds.append(y_pred)
    
len(y_preds[0])

** **输出** **

300

** **代码** **

y_preds = np.array(y_preds).T
print(y_preds.shape)
y_pred = []
for y_p in y_preds:
    y_pred.append(np.bincount(y_p.astype('int')).argmax())

print(y_pred[:10])

** **输出** **

(300, 10)

0, 0, 0, 0, 0, 1, 0, 1, 0, 0

** **代码** **

from sklearn.metrics import accuracy_score
print(accuracy_score(y_test, y_pred))

** **输出** **

0.7366666666666667

代码

class RandomForest():
    def __init__(self, n_estimators=100, min_samples_split=2, min_gain=0,
                 max_depth=float("inf"), max_features=None):
        # 树的棵树
        self.n_estimators = n_estimators
        # 树最小分裂样本数
        self.min_samples_split = min_samples_split
        # 最小增益
        self.min_gain = min_gain
        # 树最大深度
        self.max_depth = max_depth
        # 所使用最大特征数
        self.max_features = max_features

        self.trees = []
        # 基于决策树构建森林
        for _ in range(self.n_estimators):
            tree = ClassificationTree(min_samples_split=self.min_samples_split, min_impurity=self.min_gain,
                                      max_depth=self.max_depth)
            self.trees.append(tree)
            
    # 自助抽样
    def bootstrap_sampling(self, X, y):
        X_y = np.concatenate([X, y.reshape(-1,1)], axis=1)
        np.random.shuffle(X_y)
        n_samples = X.shape[0]
        sampling_subsets = []

        for _ in range(self.n_estimators):
            # 第一个随机性，行抽样
            idx1 = np.random.choice(n_samples, n_samples, replace=True)
            bootstrap_Xy = X_y[idx1, :]
            bootstrap_X = bootstrap_Xy[:, :-1]
            bootstrap_y = bootstrap_Xy[:, -1]
            sampling_subsets.append([bootstrap_X, bootstrap_y])
        return sampling_subsets
            
    # 随机森林训练
    def fit(self, X, y):
        # 对森林中每棵树训练一个双随机抽样子集
        sub_sets = self.bootstrap_sampling(X, y)
        n_features = X.shape[1]
        # 设置max_feature
        if self.max_features == None:
            self.max_features = int(np.sqrt(n_features))
        
        for i in range(self.n_estimators):
            # 第二个随机性，列抽样
            sub_X, sub_y = sub_sets[i]
            idx2 = np.random.choice(n_features, self.max_features, replace=True)
            sub_X = sub_X[:, idx2]
            self.trees[i].fit(sub_X, sub_y)
            # 保存每次列抽样的列索引，方便预测时每棵树调用
            self.trees[i].feature_indices = idx2
            print('The {}th tree is trained done...'.format(i+1))
    
    # 随机森林预测
    def predict(self, X):
        y_preds = []
        for i in range(self.n_estimators):
            idx = self.trees[i].feature_indices
            sub_X = X[:, idx]
            y_pred = self.trees[i].predict(sub_X)
            y_preds.append(y_pred)
            
        y_preds = np.array(y_preds).T
        res = []
        for j in y_preds:
            res.append(np.bincount(j.astype('int')).argmax())
        return res

** **代码** **

rf = RandomForest(n_estimators=10, max_features=15)
rf.fit(X_train, y_train)
y_pred = rf.predict(X_test)
print(accuracy_score(y_test, y_pred))

** **输出** **

The 1th tree is trained done...

The 2th tree is trained done...

The 3th tree is trained done...

The 4th tree is trained done...

The 5th tree is trained done...

The 6th tree is trained done...

The 7th tree is trained done...

The 8th tree is trained done...

The 9th tree is trained done...

The 10th tree is trained done...

** **代码** **

from sklearn.ensemble import RandomForestClassifier
clf = RandomForestClassifier(max_depth=3, random_state=0)
clf.fit(X_train, y_train)
y_pred = clf.predict(X_test)
print(accuracy_score(y_test, y_pred))

** **输出** **

0.82

** **输出** **

D:\Installation\anaconda\install\lib\site-packages\sklearn\ensemble\forest.py:246: FutureWarning: The default value of n_estimators will change from 10 in version 0.20 to 100 in 0.22.

"10 in version 0.20 to 100 in 0.22.", FutureWarning)

cart.py

import numpy as np
from utils import feature_split, calculate_gini

### 定义树结点
class TreeNode():
    def __init__(self, feature_i=None, threshold=None,
               leaf_value=None, left_branch=None, right_branch=None):
        # 特征索引
        self.feature_i = feature_i          
        # 特征划分阈值
        self.threshold = threshold 
        # 叶子节点取值
        self.leaf_value = leaf_value   
        # 左子树
        self.left_branch = left_branch     
        # 右子树
        self.right_branch = right_branch

		
### 定义二叉决策树
class BinaryDecisionTree(object):
    ### 决策树初始参数
    def __init__(self, min_samples_split=2, min_gini_impurity=999,
                 max_depth=float("inf"), loss=None):
        # 根结点
        self.root = None  
        # 节点最小分裂样本数
        self.min_samples_split = min_samples_split
        # 节点初始化基尼不纯度
        self.min_gini_impurity = min_gini_impurity
        # 树最大深度
        self.max_depth = max_depth
        # 基尼不纯度计算函数
        self.gini_impurity_calculation = None
        # 叶子节点值预测函数
        self._leaf_value_calculation = None
        # 损失函数
        self.loss = loss

    ### 决策树拟合函数
    def fit(self, X, y, loss=None):
        # 递归构建决策树
        self.root = self._build_tree(X, y)
        self.loss = None

    ### 决策树构建函数
    def _build_tree(self, X, y, current_depth=0):
        # 初始化最小基尼不纯度
        init_gini_impurity = 999
        # 初始化最佳特征索引和阈值
        best_criteria = None    
        # 初始化数据子集
        best_sets = None        
        
        if len(np.shape(y)) == 1:
            y = np.expand_dims(y, axis=1)

        # 合并输入和标签
        Xy = np.concatenate((X, y), axis=1)
        # 获取样本数和特征数
        n_samples, n_features = X.shape
        # 设定决策树构建条件
        # 训练样本数量大于节点最小分裂样本数且当前树深度小于最大深度
        if n_samples >= self.min_samples_split and current_depth <= self.max_depth:
            # 遍历计算每个特征的基尼不纯度
            for feature_i in range(n_features):
                # 获取第i特征的所有取值
                feature_values = np.expand_dims(X[:, feature_i], axis=1)
                # 获取第i个特征的唯一取值
                unique_values = np.unique(feature_values)

                # 遍历取值并寻找最佳特征分裂阈值
                for threshold in unique_values:
                    # 特征节点二叉分裂
                    Xy1, Xy2 = feature_split(Xy, feature_i, threshold)
                    # 如果分裂后的子集大小都不为0
                    if len(Xy1) > 0 and len(Xy2) > 0:
                        # 获取两个子集的标签值
                        y1 = Xy1[:, n_features:]
                        y2 = Xy2[:, n_features:]

                        # 计算基尼不纯度
                        impurity = self.impurity_calculation(y, y1, y2)

                        # 获取最小基尼不纯度
                        # 最佳特征索引和分裂阈值
                        if impurity < init_gini_impurity:
                            init_gini_impurity = impurity
                            best_criteria = {"feature_i": feature_i, "threshold": threshold}
                            best_sets = {
                                "leftX": Xy1[:, :n_features],   
                                "lefty": Xy1[:, n_features:],   
                                "rightX": Xy2[:, :n_features],  
                                "righty": Xy2[:, n_features:]   
                                }
        
        # 如果计算的最小不纯度小于设定的最小不纯度
        if init_gini_impurity < self.min_gini_impurity:
            # 分别构建左右子树
            left_branch = self._build_tree(best_sets["leftX"], best_sets["lefty"], current_depth + 1)
            right_branch = self._build_tree(best_sets["rightX"], best_sets["righty"], current_depth + 1)
            return TreeNode(feature_i=best_criteria["feature_i"], threshold=best_criteria["threshold"], left_branch=left_branch, right_branch=right_branch)

        # 计算叶子计算取值
        leaf_value = self._leaf_value_calculation(y)
        return TreeNode(leaf_value=leaf_value)

    ### 定义二叉树值预测函数
    def predict_value(self, x, tree=None):
        if tree is None:
            tree = self.root
        # 如果叶子节点已有值，则直接返回已有值
        if tree.leaf_value is not None:
            return tree.leaf_value
        # 选择特征并获取特征值
        feature_value = x[tree.feature_i]
        # 判断落入左子树还是右子树
        branch = tree.right_branch
        if isinstance(feature_value, int) or isinstance(feature_value, float):
            if feature_value >= tree.threshold:
                branch = tree.left_branch
        elif feature_value == tree.threshold:
            branch = tree.right_branch
        # 测试子集
        return self.predict_value(x, branch)

    ### 数据集预测函数
    def predict(self, X):
        y_pred = [self.predict_value(sample) for sample in X]
        return y_pred

				
class ClassificationTree(BinaryDecisionTree):
    ### 定义基尼不纯度计算过程
    def _calculate_gini_impurity(self, y, y1, y2):
        p = len(y1) / len(y)
        gini = calculate_gini(y)
        gini_impurity = p * calculate_gini(y1) + (1-p) * calculate_gini(y2)
        return gini_impurity
    
    ### 多数投票
    def _majority_vote(self, y):
        most_common = None
        max_count = 0
        for label in np.unique(y):
            # 统计多数
            count = len(y[y == label])
            if count > max_count:
                most_common = label
                max_count = count
        return most_common
    
    # 分类树拟合
    def fit(self, X, y):
        self.impurity_calculation = self._calculate_gini_impurity
        self._leaf_value_calculation = self._majority_vote
        super(ClassificationTree, self).fit(X, y)

		
### CART回归树
class RegressionTree(BinaryDecisionTree):
	# 计算方差减少量
    def _calculate_variance_reduction(self, y, y1, y2):
        var_tot = np.var(y, axis=0)
        var_y1 = np.var(y1, axis=0)
        var_y2 = np.var(y2, axis=0)
        frac_1 = len(y1) / len(y)
        frac_2 = len(y2) / len(y)
        # 计算方差减少量
        variance_reduction = var_tot - (frac_1 * var_y1 + frac_2 * var_y2)
        return sum(variance_reduction)

    # 节点值取平均
    def _mean_of_y(self, y):
        value = np.mean(y, axis=0)
        return value if len(value) > 1 else value[0]

	# 回归树拟合
    def fit(self, X, y):
        self.impurity_calculation = self._calculate_variance_reduction
        self._leaf_value_calculation = self._mean_of_y
        super(RegressionTree, self).fit(X, y)