一、布隆过滤器基本原理
1.1 数据结构与算法
python
复制
下载
class BloomFilter:
"""布隆过滤器基本实现"""
def __init__(self, n: int, p: float):
"""
初始化布隆过滤器
Args:
n: 预期插入的元素数量
p: 期望的误判率（假阳性率）
"""
# 计算最优参数
self.m = self._optimal_bit_array_size(n, p) # 位数组大小
self.k = self._optimal_hash_functions(n, self.m) # 哈希函数数量
# 初始化位数组
self.bit_array = [0] * self.m
self.size = 0
self.capacity = n
# 初始化哈希函数
self.hash_functions = self._create_hash_functions(self.k)
def _optimal_bit_array_size(self, n: int, p: float) -> int:
"""计算最优位数组大小"""
import math
# m = - (n * ln(p)) / (ln(2))^2
m = - (n * math.log(p)) / (math.log(2) ** 2)
return int(math.ceil(m))
def _optimal_hash_functions(self, n: int, m: int) -> int:
"""计算最优哈希函数数量"""
import math
# k = (m / n) * ln(2)
k = (m / n) * math.log(2)
return int(math.ceil(k))
def _create_hash_functions(self, k: int):
"""创建k个独立的哈希函数"""
import hashlib
def hash_factory(seed: int):
"""基于seed创建不同的哈希函数"""
def hash_func(data: bytes) -> int:
# 使用不同的哈希算法混合
h1 = hashlib.md5(data + str(seed).encode()).digest()
h2 = hashlib.sha256(data + str(seed*2).encode()).digest()
# 组合多个哈希值
combined = int.from_bytes(h1 + h2[:8], 'big')
return combined % self.m
return hash_func
# 生成k个不同的哈希函数
return [hash_factory(i) for i in range(k)]
def add(self, item: any) -> None:
"""添加元素到布隆过滤器"""
item_bytes = str(item).encode()
for hash_func in self.hash_functions:
position = hash_func(item_bytes)
self.bit_array[position] = 1
self.size += 1
def contains(self, item: any) -> bool:
"""检查元素是否可能存在"""
item_bytes = str(item).encode()
for hash_func in self.hash_functions:
position = hash_func(item_bytes)
if self.bit_array[position] == 0:
return False # 一定不存在
return True # 可能存在（存在误判可能）
def false_positive_rate(self) -> float:
"""计算当前误判率"""
import math
# 理论误判率公式: (1 - e^(-k*n/m))^k
if self.size == 0:
return 0.0
k = self.k
n = self.size
m = self.m
return pow(1 - pow(1 - 1/m, k*n), k)1.2 误判率数学推导
python
复制
下载
import numpy as np
import matplotlib.pyplot as plt
class BloomFilterAnalysis:
"""布隆过滤器误判率分析"""
@staticmethod
def theoretical_fpr(n: int, m: int, k: int) -> float:
"""
理论误判率计算
公式: p = (1 - e^(-k*n/m))^k
"""
return pow(1 - np.exp(-k * n / m), k)
@staticmethod
def optimal_params(n: int, p: float) -> tuple:
"""
计算给定n和期望误判率p的最优参数
返回: (m, k) - 位数组大小和哈希函数数量
"""
import math
# 最优位数组大小
m = - (n * math.log(p)) / (math.log(2) ** 2)
m = int(math.ceil(m))
# 最优哈希函数数量
k = (m / n) * math.log(2)
k = int(math.ceil(k))
return m, k
@staticmethod
def plot_fpr_vs_parameters():
"""可视化误判率与参数关系"""
# 设置参数范围
n_values = [1000, 10000, 100000] # 不同元素数量
m_over_n_ratios = np.linspace(1, 20, 50) # m/n 比例
fig, axes = plt.subplots(1, 2, figsize=(15, 6))
# 图1: 不同m/n比例下的误判率
for n in n_values:
m_values = [int(ratio * n) for ratio in m_over_n_ratios]
# 对于每个m，计算最优k和对应的误判率
fpr_values = []
for m in m_values:
k_optimal = int((m / n) * np.log(2))
fpr = BloomFilterAnalysis.theoretical_fpr(n, m, k_optimal)
fpr_values.append(fpr)
axes[0].plot(m_over_n_ratios, fpr_values, label=f'n={n}')
axes[0].set_xlabel('m/n (Bits per element)')
axes[0].set_ylabel('False Positive Rate')
axes[0].set_title('False Positive Rate vs Bits per Element')
axes[0].set_yscale('log')
axes[0].legend()
axes[0].grid(True, alpha=0.3)
# 图2: 不同k值对误判率的影响
n = 10000
m = n * 10 # 10 bits per element
k_values = range(1, 21)
fpr_values_k = [BloomFilterAnalysis.theoretical_fpr(n, m, k) for k in k_values]
# 理论最优k值
k_optimal = int((m / n) * np.log(2))
axes[1].plot(k_values, fpr_values_k, 'b-', label='FPR')
axes[1].axvline(k_optimal, color='r', linestyle='--', label=f'Optimal k={k_optimal}')
axes[1].set_xlabel('Number of Hash Functions (k)')
axes[1].set_ylabel('False Positive Rate')
axes[1].set_title(f'FPR vs Number of Hash Functions (n={n}, m={m})')
axes[1].set_yscale('log')
axes[1].legend()
axes[1].grid(True, alpha=0.3)
plt.tight_layout()
plt.show()
@staticmethod
def empirical_fpr_test(n_inserted: int, n_tested: int, m: int, k: int) -> float:
"""
经验误判率测试
"""
import random
from collections import Counter
# 创建布隆过滤器
bf = BloomFilter(n_inserted, 0.01) # 使用固定误判率初始化
# 插入n_inserted个随机元素
inserted_items = set()
for _ in range(n_inserted):
item = random.randint(0, 2**64)
bf.add(item)
inserted_items.add(item)
# 测试n_tested个新元素
false_positives = 0
for _ in range(n_tested):
# 生成不在inserted_items中的测试元素
while True:
test_item = random.randint(0, 2**64)
if test_item not in inserted_items:
break
if bf.contains(test_item):
false_positives += 1
return false_positives / n_tested二、哈希函数选择策略
2.1 哈希函数性能对比
python
复制
下载
import hashlib
import mmh3 # murmurhash3
import xxhash
import numpy as np
from typing import List, Callable
class HashFunctionBenchmark:
"""哈希函数性能基准测试"""
@staticmethod
def create_hash_functions(k: int, m: int, strategy: str = 'double_hashing') -> List[Callable]:
"""
创建k个哈希函数
Args:
k: 哈希函数数量
m: 位数组大小
strategy: 创建策略
- 'independent': 独立哈希函数
- 'double_hashing': 双重哈希
- 'enhanced_double_hashing': 增强双重哈希
"""
if strategy == 'independent':
return HashFunctionBenchmark._independent_hashes(k, m)
elif strategy == 'double_hashing':
return HashFunctionBenchmark._double_hashing_hashes(k, m)
elif strategy == 'enhanced_double_hashing':
return HashFunctionBenchmark._enhanced_double_hashing_hashes(k, m)
else:
raise ValueError(f"Unknown strategy: {strategy}")
@staticmethod
def _independent_hashes(k: int, m: int) -> List[Callable]:
"""独立哈希函数策略"""
import hashlib
def create_hash_func(seed: int):
def hash_func(data: bytes) -> int:
# 使用不同哈希算法组合
h = hashlib.sha256(data + str(seed).encode()).digest()
return int.from_bytes(h[:8], 'big') % m
return hash_func
return [create_hash_func(i) for i in range(k)]
@staticmethod
def _double_hashing_hashes(k: int, m: int) -> List[Callable]:
"""
双重哈希策略 (Kirsch & Mitzenmacher)
只需要2个独立哈希函数，生成k个哈希值
"""
import hashlib
def h1(data: bytes) -> int:
h = hashlib.md5(data).digest()
return int.from_bytes(h[:8], 'big') % m
def h2(data: bytes) -> int:
h = hashlib.sha256(data).digest()
return int.from_bytes(h[:8], 'big') % m
def create_hash_func(i: int):
def hash_func(data: bytes) -> int:
# gi(x) = h1(x) + i * h2(x) mod m
return (h1(data) + i * h2(data)) % m
return hash_func
return [create_hash_func(i) for i in range(k)]
@staticmethod
def _enhanced_double_hashing_hashes(k: int, m: int) -> List[Callable]:
"""
增强型双重哈希策略
提供更好的独立性
"""
import hashlib
def h1(data: bytes) -> int:
h = hashlib.md5(data).digest()
return int.from_bytes(h, 'big')
def h2(data: bytes) -> int:
h = hashlib.sha256(data).digest()
return int.from_bytes(h, 'big')
def create_hash_func(i: int):
def hash_func(data: bytes) -> int:
# gi(x) = (a * h1(x) + b * h2(x) + c * i) mod m
# 使用质数避免模式重复
a = 7919 # 质数
b = 7727 # 质数
c = 7577 # 质数
val = (a * h1(data) + b * h2(data) + c * i)
return val % m
return hash_func
return [create_hash_func(i) for i in range(k)]
@staticmethod
def benchmark_hash_functions():
"""哈希函数性能基准测试"""
import time
import random
# 测试参数
m = 1000000 # 位数组大小
k = 10 # 哈希函数数量
n_tests = 100000 # 测试次数
strategies = ['independent', 'double_hashing', 'enhanced_double_hashing']
results = {}
for strategy in strategies:
print(f"\n测试策略: {strategy}")
# 创建哈希函数
hash_funcs = HashFunctionBenchmark.create_hash_functions(k, m, strategy)
# 性能测试
test_data = [str(random.randint(0, 2**64)).encode() for _ in range(n_tests)]
# 计算时间
start_time = time.time()
for data in test_data:
for hash_func in hash_funcs:
_ = hash_func(data)
execution_time = time.time() - start_time
# 测试独立性（碰撞率）
collision_test = HashFunctionBenchmark._test_collision_rate(hash_funcs, m, n_tests)
results[strategy] = {
'execution_time': execution_time,
'hashes_per_second': n_tests * k / execution_time,
'collision_rate': collision_test['collision_rate'],
'uniformity_score': collision_test['uniformity_score']
}
print(f"执行时间: {execution_time:.3f}秒")
print(f"哈希速度: {results[strategy]['hashes_per_second']:,.0f} 哈希/秒")
print(f"碰撞率: {collision_test['collision_rate']:.6f}")
print(f"均匀性得分: {collision_test['uniformity_score']:.3f}")
return results
@staticmethod
def _test_collision_rate(hash_funcs: List[Callable], m: int, n_samples: int) -> dict:
"""测试哈希函数的碰撞率和均匀性"""
import random
from collections import Counter
# 测试不同输入
test_inputs = [str(random.randint(0, 2**64)).encode() for _ in range(n_samples)]
all_positions = []
position_counts = Counter()
for data in test_inputs:
positions = []
for hash_func in hash_funcs:
pos = hash_func(data)
positions.append(pos)
position_counts[pos] += 1
all_positions.append(tuple(sorted(positions)))
# 计算碰撞率（不同输入产生相同位置集合的比例）
total_combinations = len(all_positions)
unique_combinations = len(set(all_positions))
collision_rate = 1 - (unique_combinations / total_combinations)
# 计算均匀性（卡方检验）
expected_count = (n_samples * len(hash_funcs)) / m
chi_square = 0
for pos in range(m):
observed = position_counts.get(pos, 0)
if expected_count > 0:
chi_square += ((observed - expected_count) ** 2) / expected_count
# 均匀性得分（越小越好）
uniformity_score = chi_square / m
return {
'collision_rate': collision_rate,
'uniformity_score': uniformity_score,
'position_distribution': position_counts
}篇幅限制下面就只能给大家展示小册部分内容了。整理了一份核心面试笔记包括了：Java面试、Spring、JVM、MyBatis、Redis、MySQL、并发编程、微服务、Linux、Springboot、SpringCloud、MQ、Kafc
需要全套面试笔记及答案
【点击此处即可/免费获取】
2.2 专业级哈希函数实现
python
复制
下载
import struct
import zlib
class ProfessionalHashFunctions:
"""专业级哈希函数集合"""
@staticmethod
def murmur_hash3_32(data: bytes, seed: int = 0) -> int:
"""
MurmurHash3 32位实现
高性能，低碰撞率
"""
def fmix32(h: int) -> int:
h ^= h >> 16
h = (h * 0x85ebca6b) & 0xFFFFFFFF
h ^= h >> 13
h = (h * 0xc2b2ae35) & 0xFFFFFFFF
h ^= h >> 16
return h
length = len(data)
nblocks = length // 4
h1 = seed
# 主体
for i in range(nblocks):
k1 = struct.unpack_from('<I', data, i*4)[0]
k1 = (k1 * 0xcc9e2d51) & 0xFFFFFFFF
k1 = ((k1 << 15) | (k1 >> 17)) & 0xFFFFFFFF # ROTL32
k1 = (k1 * 0x1b873593) & 0xFFFFFFFF
h1 ^= k1
h1 = ((h1 << 13) | (h1 >> 19)) & 0xFFFFFFFF # ROTL32
h1 = (h1 * 5 + 0xe6546b64) & 0xFFFFFFFF
# 尾部
tail = data[nblocks*4:]
k1 = 0
if len(tail) >= 3:
k1 ^= tail[2] << 16
if len(tail) >= 2:
k1 ^= tail[1] << 8
if len(tail) >= 1:
k1 ^= tail[0]
if len(tail) > 0:
k1 = (k1 * 0xcc9e2d51) & 0xFFFFFFFF
k1 = ((k1 << 15) | (k1 >> 17)) & 0xFFFFFFFF
k1 = (k1 * 0x1b873593) & 0xFFFFFFFF
h1 ^= k1
# 最终混合
h1 ^= length
h1 = fmix32(h1)
return h1
@staticmethod
def fnv_1a(data: bytes) -> int:
"""
FNV-1a 哈希算法
简单快速，适合短字符串
"""
FNV_prime = 0x01000193
FNV_offset_basis = 0x811c9dc5
hash_val = FNV_offset_basis
for byte in data:
hash_val ^= byte
hash_val = (hash_val * FNV_prime) & 0xFFFFFFFF
return hash_val
@staticmethod
def jenkins_one_at_a_time(data: bytes) -> int:
"""
Jenkins one-at-a-time 哈希
优秀的分布特性
"""
hash_val = 0
for byte in data:
hash_val += byte
hash_val += (hash_val << 10)
hash_val ^= (hash_val >> 6)
hash_val += (hash_val << 3)
hash_val ^= (hash_val >> 11)
hash_val += (hash_val << 15)
return hash_val & 0xFFFFFFFF
@staticmethod
def xxhash_32(data: bytes, seed: int = 0) -> int:
"""
xxHash 32位实现
极高性能
"""
PRIME32_1 = 0x9E3779B1
PRIME32_2 = 0x85EBCA77
PRIME32_3 = 0xC2B2AE3D
PRIME32_4 = 0x27D4EB2F
PRIME32_5 = 0x165667B1
def rotate_left(x: int, r: int) -> int:
return ((x << r) | (x >> (32 - r))) & 0xFFFFFFFF
length = len(data)
h32 = seed + PRIME32_5
if length >= 16:
limit = length - 16
v1 = seed + PRIME32_1 + PRIME32_2
v2 = seed + PRIME32_2
v3 = seed
v4 = seed - PRIME32_1
idx = 0
while idx <= limit:
v1 = (v1 + struct.unpack_from('<I', data, idx)[0] * PRIME32_2) & 0xFFFFFFFF
v1 = rotate_left(v1, 13)
v1 = (v1 * PRIME32_1) & 0xFFFFFFFF
idx += 4
v2 = (v2 + struct.unpack_from('<I', data, idx)[0] * PRIME32_2) & 0xFFFFFFFF
v2 = rotate_left(v2, 13)
v2 = (v2 * PRIME32_1) & 0xFFFFFFFF
idx += 4
v3 = (v3 + struct.unpack_from('<I', data, idx)[0] * PRIME32_2) & 0xFFFFFFFF
v3 = rotate_left(v3, 13)
v3 = (v3 * PRIME32_1) & 0xFFFFFFFF
idx += 4
v4 = (v4 + struct.unpack_from('<I', data, idx)[0] * PRIME32_2) & 0xFFFFFFFF
v4 = rotate_left(v4, 13)
v4 = (v4 * PRIME32_1) & 0xFFFFFFFF
idx += 4
h32 = rotate_left(v1, 1) + rotate_left(v2, 7) + \
rotate_left(v3, 12) + rotate_left(v4, 18)
else:
h32 = seed + PRIME32_5
h32 += length
# 处理剩余字节
idx = length & -4
while idx < length:
h32 = (h32 + data[idx] * PRIME32_5) & 0xFFFFFFFF
h32 = rotate_left(h32, 11)
h32 = (h32 * PRIME32_1) & 0xFFFFFFFF
idx += 1
# 最终混合
h32 ^= h32 >> 15
h32 = (h32 * PRIME32_2) & 0xFFFFFFFF
h32 ^= h32 >> 13
h32 = (h32 * PRIME32_3) & 0xFFFFFFFF
h32 ^= h32 >> 16
return h32
@staticmethod
def city_hash(data: bytes, seed: int = 0) -> int:
"""
CityHash 简化版
谷歌开发，针对字符串优化
"""
length = len(data)
if length <= 4:
# 小字符串特殊处理
h = seed
for byte in data:
h = (h * 0x5bd1e995) & 0xFFFFFFFF
h ^= byte
return h
# 中等长度字符串
a = seed
b = length * 0x5bd1e995
c = b ^ (b >> 24)
idx = 0
while idx + 4 <= length:
k = struct.unpack_from('<I', data, idx)[0]
k = (k * 0x5bd1e995) & 0xFFFFFFFF
k ^= k >> 24
k = (k * 0x5bd1e995) & 0xFFFFFFFF
a = (a * 0x5bd1e995) & 0xFFFFFFFF
a ^= k
idx += 4
# 处理尾部
tail = data[idx:]
if len(tail) > 0:
k = 0
for i, byte in enumerate(tail):
k ^= byte << (i * 8)
k = (k * 0x5bd1e995) & 0xFFFFFFFF
a ^= k
b ^= a
b = (b * 0x5bd1e995) & 0xFFFFFFFF
c ^= b
c = (c * 0x5bd1e995) & 0xFFFFFFFF
return c三、误判率优化策略
3.1 自适应布隆过滤器
python
复制
下载
class AdaptiveBloomFilter:
"""
自适应布隆过滤器
根据实际插入情况动态调整参数
"""
def __init__(self, initial_n: int, target_fpr: float = 0.01):
self.target_fpr = target_fpr
self.inserted_count = 0
# 初始参数
self.m, self.k = BloomFilterAnalysis.optimal_params(initial_n, target_fpr)
# 位数组
self.bit_array = [0] * self.m
# 哈希函数（使用增强双重哈希）
self.hash_funcs = HashFunctionBenchmark.create_hash_functions(
self.k, self.m, 'enhanced_double_hashing'
)
# 监控指标
self.fpr_history = []
self.resize_history = []
def add(self, item: any) -> None:
"""添加元素，必要时动态调整"""
# 检查是否需要扩容
if self._needs_resize():
self._resize_filter()
# 执行添加
item_bytes = str(item).encode()
for hash_func in self.hash_funcs:
position = hash_func(item_bytes)
self.bit_array[position] = 1
self.inserted_count += 1
# 监控实际误判率
if self.inserted_count % 1000 == 0:
current_fpr = self.estimate_current_fpr()
self.fpr_history.append((self.inserted_count, current_fpr))
def contains(self, item: any) -> bool:
"""检查元素是否存在"""
item_bytes = str(item).encode()
for hash_func in self.hash_funcs:
position = hash_func(item_bytes)
if self.bit_array[position] == 0:
return False
return True
def _needs_resize(self) -> bool:
"""判断是否需要扩容"""
# 如果当前插入数量超过容量的150%
if self.inserted_count > 1.5 * (self.m / (self.k * 1.44)):
return True
# 如果实际误判率超过目标值
if self.estimate_current_fpr() > self.target_fpr * 1.5:
return True
return False
def _resize_filter(self) -> None:
"""动态调整过滤器大小"""
print(f"正在扩容: 当前大小={self.m}, 已插入={self.inserted_count}")
# 计算新参数（保持目标误判率）
new_n = self.inserted_count * 2 # 翻倍容量
new_m, new_k = BloomFilterAnalysis.optimal_params(new_n, self.target_fpr)
# 创建新位数组
new_bit_array = [0] * new_m
# 需要重新哈希所有已存在元素（可选）
# 在实际系统中，可能需要记录原始元素或使用计数布隆过滤器
# 更新参数
self.m = new_m
self.k = new_k
self.bit_array = new_bit_array
# 重新创建哈希函数
self.hash_funcs = HashFunctionBenchmark.create_hash_functions(
self.k, self.m, 'enhanced_double_hashing'
)
self.resize_history.append((self.inserted_count, self.m, self.k))
def estimate_current_fpr(self) -> float:
"""估计当前误判率"""
# 理论公式
import math
# 计算实际填充率
ones_count = sum(self.bit_array)
fill_ratio = ones_count / self.m
# 使用公式: p = fill_ratio^k
estimated_fpr = fill_ratio ** self.k
return estimated_fpr
def get_statistics(self) -> dict:
"""获取统计信息"""
ones_count = sum(self.bit_array)
fill_ratio = ones_count / self.m
return {
'bit_array_size': self.m,
'hash_functions': self.k,
'inserted_count': self.inserted_count,
'bits_per_element': self.m / max(1, self.inserted_count),
'fill_ratio': fill_ratio,
'estimated_fpr': self.estimate_current_fpr(),
'resize_count': len(self.resize_history),
'target_fpr': self.target_fpr
}3.2 可扩展布隆过滤器（Scalable Bloom Filter）
python
复制
下载
class ScalableBloomFilter:
"""
可扩展布隆过滤器
通过多个布隆过滤器层实现动态扩展
"""
def __init__(self, initial_capacity: int, error_rate: float = 0.01,
growth_factor: float = 2.0, tightening_ratio: float = 0.9):
"""
Args:
initial_capacity: 初始容量
error_rate: 目标误判率
growth_factor: 增长因子
tightening_ratio: 每层误判率收紧比例
"""
self.error_rate = error_rate
self.growth_factor = growth_factor
self.tightening_ratio = tightening_ratio
self.filters = [] # 布隆过滤器列表
self.current_filter = None
# 创建第一层
self._add_new_filter(initial_capacity)
# 统计信息
self.total_inserted = 0
def _add_new_filter(self, capacity: int):
"""添加新的布隆过滤器层"""
# 计算本层的误判率
if len(self.filters) == 0:
layer_error_rate = self.error_rate
else:
# 每层误判率递减
layer_error_rate = self.error_rate * (self.tightening_ratio ** len(self.filters))
# 创建新的布隆过滤器
new_filter = BloomFilter(capacity, layer_error_rate)
# 添加到列表
self.filters.append(new_filter)
self.current_filter = new_filter
print(f"添加第{len(self.filters)}层过滤器: 容量={capacity}, 目标误判率={layer_error_rate:.6f}")
def add(self, item: any):
"""添加元素"""
# 添加到当前过滤器
self.current_filter.add(item)
self.total_inserted += 1
# 检查当前过滤器是否需要扩展
if self.current_filter.size >= self.current_filter.capacity * 0.8:
# 创建新的过滤器
new_capacity = int(self.current_filter.capacity * self.growth_factor)
self._add_new_filter(new_capacity)
def contains(self, item: any) -> bool:
"""检查元素是否存在"""
# 从最新的过滤器开始检查
for bf in reversed(self.filters):
if bf.contains(item):
return True
return False
def overall_false_positive_rate(self) -> float:
"""计算整体误判率"""
# 整体误判率 = 1 - Π(1 - pi)
# 其中pi是第i层的误判率
prob_not_fp = 1.0
for bf in self.filters:
fpr = bf.false_positive_rate()
prob_not_fp *= (1 - fpr)
return 1 - prob_not_fp
def get_statistics(self) -> dict:
"""获取统计信息"""
stats = {
'total_layers': len(self.filters),
'total_inserted': self.total_inserted,
'overall_fpr': self.overall_false_positive_rate(),
'target_fpr': self.error_rate,
'layers': []
}
for i, bf in enumerate(self.filters):
layer_stats = {
'layer': i + 1,
'capacity': bf.capacity,
'size': bf.size,
'bit_array_size': bf.m,
'hash_functions': bf.k,
'current_fpr': bf.false_positive_rate(),
'fill_ratio': sum(bf.bit_array) / bf.m
}
stats['layers'].append(layer_stats)
return stats
def memory_usage(self) -> int:
"""计算总内存使用（位）"""
total_bits = 0
for bf in self.filters:
total_bits += bf.m
return total_bits四、计数布隆过滤器
python
复制
下载
class CountingBloomFilter:
"""
计数布隆过滤器
支持元素删除操作
"""
def __init__(self, n: int, p: float, counter_size: int = 4):
"""
Args:
n: 预期元素数量
p: 目标误判率
counter_size: 计数器大小（位）
"""
# 基本布隆过滤器参数
self.m = BloomFilterAnalysis.optimal_params(n, p)[0]
self.k = BloomFilterAnalysis.optimal_params(n, p)[1]
# 计数数组（每个位置使用counter_size位的计数器）
self.counters = [0] * self.m
self.counter_size = counter_size
self.counter_max = (1 << counter_size) - 1 # 计数器最大值
# 哈希函数
self.hash_funcs = HashFunctionBenchmark.create_hash_functions(
self.k, self.m, 'enhanced_double_hashing'
)
self.size = 0
def add(self, item: any) -> bool:
"""添加元素"""
item_bytes = str(item).encode()
positions = []
# 获取所有哈希位置
for hash_func in self.hash_funcs:
pos = hash_func(item_bytes)
positions.append(pos)
# 增加计数器（不超过最大值）
if self.counters[pos] < self.counter_max:
self.counters[pos] += 1
self.size += 1
# 返回是否成功添加（如果计数器溢出则失败）
success = all(self.counters[pos] <= self.counter_max for pos in positions)
return success
def remove(self, item: any) -> bool:
"""删除元素"""
if not self.contains(item):
return False # 元素不存在
item_bytes = str(item).encode()
positions = []
for hash_func in self.hash_funcs:
pos = hash_func(item_bytes)
positions.append(pos)
# 减少计数器（不低于0）
if self.counters[pos] > 0:
self.counters[pos] -= 1
self.size -= 1
return True
def contains(self, item: any) -> bool:
"""检查元素是否存在"""
item_bytes = str(item).encode()
for hash_func in self.hash_funcs:
pos = hash_func(item_bytes)
if self.counters[pos] == 0:
return False
return True
def false_positive_rate(self) -> float:
"""计算当前误判率"""
import math
# 计算有效填充率（计数器>0的位置比例）
filled_positions = sum(1 for c in self.counters if c > 0)
fill_ratio = filled_positions / self.m
return fill_ratio ** self.k
def get_statistics(self) -> dict:
"""获取统计信息"""
filled_positions = sum(1 for c in self.counters if c > 0)
overflow_count = sum(1 for c in self.counters if c >= self.counter_max)
counter_distribution = {}
for c in self.counters:
counter_distribution[c] = counter_distribution.get(c, 0) + 1
return {
'bit_array_size': self.m,
'hash_functions': self.k,
'inserted_count': self.size,
'counter_size_bits': self.counter_size,
'fill_ratio': filled_positions / self.m,
'overflow_positions': overflow_count,
'estimated_fpr': self.false_positive_rate(),
'counter_distribution': counter_distribution,
'total_memory_bits': self.m * self.counter_size
}
def compression_ratio(self) -> float:
"""计算压缩率（相对于标准布隆过滤器）"""
standard_bits = self.m # 标准布隆过滤器使用1位/位置
counting_bits = self.m * self.counter_size
return counting_bits / standard_bits篇幅限制下面就只能给大家展示小册部分内容了。整理了一份核心面试笔记包括了：Java面试、Spring、JVM、MyBatis、Redis、MySQL、并发编程、微服务、Linux、Springboot、SpringCloud、MQ、Kafc
需要全套面试笔记及答案
【点击此处即可/免费获取】
五、布隆过滤器变种
5.1 分块布隆过滤器（Blocked Bloom Filter）
python
复制
下载
import math
class BlockedBloomFilter:
"""
分块布隆过滤器
将位数组分成小块，每个块有自己的哈希函数
提高缓存局部性和并行性能
"""
def __init__(self, n: int, p: float, block_size: int = 256):
"""
Args:
n: 预期元素数量
p: 目标误判率
block_size: 每个块的大小（位）
"""
self.block_size = block_size
# 计算总位数
self.m = BloomFilterAnalysis.optimal_params(n, p)[0]
# 计算块数量
self.num_blocks = math.ceil(self.m / block_size)
# 实际总位数（补齐到块整数倍）
self.m = self.num_blocks * block_size
# 每个块的哈希函数数量
self.k_per_block = 3 # 通常每个块使用3-4个哈希函数
# 初始化块
self.blocks = []
for _ in range(self.num_blocks):
# 每个块是一个小的位数组
block = {
'bits': [0] * block_size,
'hash_seeds': self._generate_hash_seeds(self.k_per_block)
}
self.blocks.append(block)
self.size = 0
def _generate_hash_seeds(self, k: int):
"""为每个块生成独立的哈希种子"""
import random
return [random.randint(1, 2**32) for _ in range(k)]
def _block_hash(self, item_bytes: bytes, block_index: int) -> list:
"""计算元素在指定块内的哈希位置"""
positions = []
# 使用块的哈希种子
seeds = self.blocks[block_index]['hash_seeds']
for seed in seeds:
# 使用种子混合哈希
combined_hash = hash(item_bytes) ^ seed
# 计算块内位置
pos = combined_hash % self.block_size
positions.append(pos)
return positions
def add(self, item: any):
"""添加元素"""
item_bytes = str(item).encode()
# 选择目标块（使用一个全局哈希函数）
block_index = hash(item_bytes) % self.num_blocks
# 在选定的块内设置位
positions = self._block_hash(item_bytes, block_index)
for pos in positions:
self.blocks[block_index]['bits'][pos] = 1
self.size += 1
def contains(self, item: any) -> bool:
"""检查元素是否存在"""
item_bytes = str(item).encode()
# 选择目标块
block_index = hash(item_bytes) % self.num_blocks
# 检查块内的所有位
positions = self._block_hash(item_bytes, block_index)
for pos in positions:
if self.blocks[block_index]['bits'][pos] == 0:
return False
return True
def false_positive_rate(self) -> float:
"""计算误判率"""
# 统计每个块的填充率
block_fill_ratios = []
for block in self.blocks:
filled = sum(block['bits'])
fill_ratio = filled / self.block_size
block_fill_ratios.append(fill_ratio)
# 平均填充率
avg_fill_ratio = sum(block_fill_ratios) / len(block_fill_ratios)
# 误判率 = (平均填充率)^k
return avg_fill_ratio ** self.k_per_block
def get_statistics(self) -> dict:
"""获取统计信息"""
block_fill_ratios = []
for block in self.blocks:
filled = sum(block['bits'])
block_fill_ratios.append(filled / self.block_size)
import numpy as np
return {
'total_blocks': self.num_blocks,
'block_size': self.block_size,
'total_bits': self.m,
'inserted_count': self.size,
'hash_per_block': self.k_per_block,
'avg_block_fill_ratio': np.mean(block_fill_ratios),
'std_block_fill_ratio': np.std(block_fill_ratios),
'estimated_fpr': self.false_positive_rate()
}5.2 压缩布隆过滤器（Compressed Bloom Filter）
python
复制
下载
import zlib
import pickle
class CompressedBloomFilter:
"""
压缩布隆过滤器
在传输或存储时压缩位数组
"""
def __init__(self, n: int, p: float, compression_level: int = 6):
"""
Args:
n: 预期元素数量
p: 目标误判率
compression_level: 压缩级别 (1-9)
"""
self.m, self.k = BloomFilterAnalysis.optimal_params(n, p)
# 位数组
self.bit_array = bytearray(self.m // 8 + 1) # 使用字节数组
# 哈希函数
self.hash_funcs = HashFunctionBenchmark.create_hash_functions(
self.k, self.m, 'enhanced_double_hashing'
)
self.compression_level = compression_level
self.size = 0
# 压缩统计
self.compression_stats = {
'original_size': 0,
'compressed_size': 0,
'compression_ratio': 1.0
}
def _set_bit(self, position: int):
"""设置位数组中指定位为1"""
byte_index = position // 8
bit_index = position % 8
self.bit_array[byte_index] |= (1 << bit_index)
def _get_bit(self, position: int) -> bool:
"""获取位数组中指定位的值"""
byte_index = position // 8
bit_index = position % 8
return (self.bit_array[byte_index] & (1 << bit_index)) != 0
def add(self, item: any):
"""添加元素"""
item_bytes = str(item).encode()
for hash_func in self.hash_funcs:
position = hash_func(item_bytes)
self._set_bit(position)
self.size += 1
def contains(self, item: any) -> bool:
"""检查元素是否存在"""
item_bytes = str(item).encode()
for hash_func in self.hash_funcs:
position = hash_func(item_bytes)
if not self._get_bit(position):
return False
return True
def compress(self) -> bytes:
"""压缩布隆过滤器"""
# 序列化数据结构
data = pickle.dumps({
'bit_array': self.bit_array,
'm': self.m,
'k': self.k,
'size': self.size
})
self.compression_stats['original_size'] = len(data)
# 使用zlib压缩
compressed = zlib.compress(data, level=self.compression_level)
self.compression_stats['compressed_size'] = len(compressed)
self.compression_stats['compression_ratio'] = (
self.compression_stats['original_size'] /
self.compression_stats['compressed_size']
)
return compressed
@classmethod
def decompress(cls, compressed_data: bytes):
"""从压缩数据恢复布隆过滤器"""
# 解压缩
data = zlib.decompress(compressed_data)
state = pickle.loads(data)
# 创建新实例
# 估计原始参数
n = state['size'] * 2 # 估计原始容量
instance = cls(n, 0.01) # 使用默认误判率
# 恢复状态
instance.bit_array = state['bit_array']
instance.m = state['m']
instance.k = state['k']
instance.size = state['size']
# 重新创建哈希函数（需要相同参数）
instance.hash_funcs = HashFunctionBenchmark.create_hash_functions(
instance.k, instance.m, 'enhanced_double_hashing'
)
return instance
def get_statistics(self) -> dict:
"""获取统计信息"""
# 计算填充率
total_bits = self.m
filled_bits = 0
for i in range(self.m):
if self._get_bit(i):
filled_bits += 1
fill_ratio = filled_bits / total_bits
estimated_fpr = fill_ratio ** self.k
stats = {
'total_bits': self.m,
'hash_functions': self.k,
'inserted_count': self.size,
'fill_ratio': fill_ratio,
'estimated_fpr': estimated_fpr,
'memory_bytes': len(self.bit_array),
'compression_stats': self.compression_stats.copy()
}
return stats六、性能测试与基准比较
python
复制
下载
import time
import random
from dataclasses import dataclass
from typing import List, Dict
@dataclass
class BenchmarkResult:
"""基准测试结果"""
filter_type: str
insert_time: float
query_time: float
memory_usage: int
actual_fpr: float
theoretical_fpr: float
config: Dict
class BloomFilterBenchmark:
"""布隆过滤器性能基准测试套件"""
def __init__(self, dataset_size: int = 100000, test_size: int = 10000):
self.dataset_size = dataset_size
self.test_size = test_size
# 生成测试数据集
self.positive_set = set()
self.negative_set = set()
self._generate_datasets()
def _generate_datasets(self):
"""生成测试数据集"""
print("生成测试数据集...")
# 生成插入数据集（正例）
while len(self.positive_set) < self.dataset_size:
item = random.randint(0, 2**64)
self.positive_set.add(item)
# 生成测试数据集（负例）
while len(self.negative_set) < self.test_size:
item = random.randint(0, 2**64)
if item not in self.positive_set:
self.negative_set.add(item)
print(f"数据集生成完成: {self.dataset_size}个正例, {self.test_size}个负例")
def benchmark_standard(self, target_fpr: float = 0.01) -> BenchmarkResult:
"""测试标准布隆过滤器"""
print("\n测试标准布隆过滤器...")
# 创建过滤器
bf = BloomFilter(self.dataset_size, target_fpr)
# 插入性能测试
insert_start = time.time()
for item in list(self.positive_set)[:self.dataset_size]:
bf.add(item)
insert_time = time.time() - insert_start
# 查询性能测试
query_start = time.time()
false_positives = 0
for item in self.negative_set:
if bf.contains(item):
false_positives += 1
query_time = time.time() - query_start
# 计算实际误判率
actual_fpr = false_positives / len(self.negative_set)
result = BenchmarkResult(
filter_type="Standard Bloom Filter",
insert_time=insert_time,
query_time=query_time,
memory_usage=bf.m,
actual_fpr=actual_fpr,
theoretical_fpr=target_fpr,
config={
'm': bf.m,
'k': bf.k,
'bits_per_element': bf.m / self.dataset_size
}
)
return result
def benchmark_counting(self, target_fpr: float = 0.01, counter_size: int = 4) -> BenchmarkResult:
"""测试计数布隆过滤器"""
print("\n测试计数布隆过滤器...")
# 创建过滤器
cbf = CountingBloomFilter(self.dataset_size, target_fpr, counter_size)
# 插入性能测试
insert_start = time.time()
for item in list(self.positive_set)[:self.dataset_size]:
cbf.add(item)
insert_time = time.time() - insert_start
# 查询性能测试
query_start = time.time()
false_positives = 0
for item in self.negative_set:
if cbf.contains(item):
false_positives += 1
query_time = time.time() - query_start
# 计算实际误判率
actual_fpr = false_positives / len(self.negative_set)
# 内存使用（计数位）
memory_bits = cbf.m * counter_size
result = BenchmarkResult(
filter_type=f"Counting Bloom Filter (counter_size={counter_size})",
insert_time=insert_time,
query_time=query_time,
memory_usage=memory_bits,
actual_fpr=actual_fpr,
theoretical_fpr=target_fpr,
config={
'm': cbf.m,
'k': cbf.k,
'counter_size': counter_size,
'bits_per_element': memory_bits / self.dataset_size
}
)
return result
def benchmark_scalable(self, initial_capacity: int = 10000,
target_fpr: float = 0.01) -> BenchmarkResult:
"""测试可扩展布隆过滤器"""
print("\n测试可扩展布隆过滤器...")
# 创建过滤器
sbf = ScalableBloomFilter(initial_capacity, target_fpr)
# 插入性能测试
insert_start = time.time()
for item in list(self.positive_set)[:self.dataset_size]:
sbf.add(item)
insert_time = time.time() - insert_start
# 查询性能测试
query_start = time.time()
false_positives = 0
for item in self.negative_set:
if sbf.contains(item):
false_positives += 1
query_time = time.time() - query_start
# 计算实际误判率
actual_fpr = false_positives / len(self.negative_set)
# 总内存使用
total_memory = sbf.memory_usage()
result = BenchmarkResult(
filter_type="Scalable Bloom Filter",
insert_time=insert_time,
query_time=query_time,
memory_usage=total_memory,
actual_fpr=actual_fpr,
theoretical_fpr=target_fpr,
config={
'initial_capacity': initial_capacity,
'growth_factor': sbf.growth_factor,
'layers': len(sbf.filters),
'bits_per_element': total_memory / self.dataset_size
}
)
return result
def benchmark_blocked(self, target_fpr: float = 0.01,
block_size: int = 256) -> BenchmarkResult:
"""测试分块布隆过滤器"""
print("\n测试分块布隆过滤器...")
# 创建过滤器
bbf = BlockedBloomFilter(self.dataset_size, target_fpr, block_size)
# 插入性能测试
insert_start = time.time()
for item in list(self.positive_set)[:self.dataset_size]:
bbf.add(item)
insert_time = time.time() - insert_start
# 查询性能测试
query_start = time.time()
false_positives = 0
for item in self.negative_set:
if bbf.contains(item):
false_positives += 1
query_time = time.time() - query_start
# 计算实际误判率
actual_fpr = false_positives / len(self.negative_set)
result = BenchmarkResult(
filter_type=f"Blocked Bloom Filter (block_size={block_size})",
insert_time=insert_time,
query_time=query_time,
memory_usage=bbf.m,
actual_fpr=actual_fpr,
theoretical_fpr=target_fpr,
config={
'm': bbf.m,
'blocks': bbf.num_blocks,
'block_size': block_size,
'hash_per_block': bbf.k_per_block,
'bits_per_element': bbf.m / self.dataset_size
}
)
return result
def run_comprehensive_benchmark(self):
"""运行全面的性能基准测试"""
print("=" * 60)
print("布隆过滤器全面性能基准测试")
print("=" * 60)
results = []
# 测试不同误判率目标
target_fprs = [0.001, 0.01, 0.05, 0.1]
for fpr in target_fprs:
print(f"\n目标误判率: {fpr}")
# 测试标准布隆过滤器
result_std = self.benchmark_standard(fpr)
results.append(result_std)
# 测试计数布隆过滤器
result_cnt = self.benchmark_counting(fpr, counter_size=4)
results.append(result_cnt)
# 测试可扩展布隆过滤器
result_scalable = self.benchmark_scalable(self.dataset_size//10, fpr)
results.append(result_scalable)
# 生成报告
self._generate_report(results)
return results
def _generate_report(self, results: List[BenchmarkResult]):
"""生成性能报告"""
print("\n" + "=" * 80)
print("性能基准测试报告")
print("=" * 80)
print("\n性能对比:")
print(f"{'过滤器类型':<35} {'插入时间(s)':<12} {'查询时间(s)':<12} "
f"{'内存(bits)':<12} {'实际FPR':<10} {'理论FPR':<10} {'bits/elem':<10}")
print("-" * 110)
for result in results:
bits_per_elem = result.memory_usage / self.dataset_size
print(f"{result.filter_type:<35} {result.insert_time:<12.4f} "
f"{result.query_time:<12.4f} {result.memory_usage:<12,d} "
f"{result.actual_fpr:<10.6f} {result.theoretical_fpr:<10.3f} "
f"{bits_per_elem:<10.2f}")
# 分析总结
print("\n" + "=" * 80)
print("关键发现:")
# 按过滤器类型分组
grouped_results = {}
for result in results:
key = result.filter_type.split('(')[0].strip()
if key not in grouped_results:
grouped_results[key] = []
grouped_results[key].append(result)
for filter_type, filter_results in grouped_results.items():
print(f"\n{filter_type}:")
# 找到最佳配置
best_memory = min(r.memory_usage for r in filter_results)
best_fpr = min(r.actual_fpr for r in filter_results)
avg_insert_time = sum(r.insert_time for r in filter_results) / len(filter_results)
avg_query_time = sum(r.query_time for r in filter_results) / len(filter_results)
print(f" 最佳内存使用: {best_memory:,} bits")
print(f" 最佳误判率: {best_fpr:.6f}")
print(f" 平均插入时间: {avg_insert_time:.4f}秒")
print(f" 平均查询时间: {avg_query_time:.4f}秒")
print("\n" + "=" * 80)
print("推荐配置:")
# 根据需求推荐配置
print("\n1. 内存敏感场景:")
print(" - 使用标准布隆过滤器，目标FPR=0.01")
print(" - bits/element ≈ 9.6, 内存效率最高")
print("\n2. 需要删除操作的场景:")
print(" - 使用计数布隆过滤器，counter_size=4")
print(" - 注意counter溢出的可能性")
print("\n3. 数据量不确定的场景:")
print(" - 使用可扩展布隆过滤器，initial_capacity=预计最小数据量")
print(" - 自动扩展，避免重新构建")
print("\n4. 高性能查询场景:")
print(" - 使用分块布隆过滤器，block_size=256")
print(" - 提高缓存局部性，适合并行查询")
print("\n5. 网络传输场景:")
print(" - 使用压缩布隆过滤器，compression_level=6")
print(" - 传输前压缩，节省带宽")七、实际应用示例
7.1 Web应用中的URL去重
python
复制
下载
class URLDeduplicator:
"""使用布隆过滤器的URL去重器"""
def __init__(self, capacity: int = 1000000, target_fpr: float = 0.001):
"""
初始化URL去重器
Args:
capacity: 预期处理的URL数量
target_fpr: 可接受的误判率
"""
self.capacity = capacity
# 使用可扩展布隆过滤器处理不确定的数据量
self.bloom_filter = ScalableBloomFilter(
initial_capacity=capacity // 10,
error_rate=target_fpr,
growth_factor=2.0
)
# URL规范化器
from urllib.parse import urlparse, urlunparse
self.normalize_url = lambda url: self._normalize_url(url)
# 统计信息
self.stats = {
'total_urls': 0,
'unique_urls': 0,
'duplicate_urls': 0,
'false_positives_estimated': 0
}
def _normalize_url(self, url: str) -> str:
"""URL规范化"""
from urllib.parse import urlparse, urlunparse
try:
parsed = urlparse(url)
# 标准化协议
scheme = parsed.scheme.lower()
# 标准化主机名
netloc = parsed.netloc.lower()
if ':' in netloc:
# 移除默认端口
hostname, port = netloc.split(':', 1)
if (scheme == 'http' and port == '80') or \
(scheme == 'https' and port == '443'):
netloc = hostname
# 标准化路径（移除尾随斜杠）
path = parsed.path.rstrip('/')
# 标准化查询参数（可选排序）
query = parsed.query
# 重建URL
normalized = urlunparse((scheme, netloc, path,
parsed.params, query, parsed.fragment))
return normalized
except Exception as e:
print(f"URL规范化失败: {url}, 错误: {e}")
return url
def is_duplicate(self, url: str) -> bool:
"""检查URL是否重复"""
normalized_url = self.normalize_url(url)
# 检查布隆过滤器
if self.bloom_filter.contains(normalized_url):
# 可能存在（需要进一步验证）
return True
else:
# 一定不存在
self.bloom_filter.add(normalized_url)
self.stats['unique_urls'] += 1
return False
def process_urls(self, urls: list) -> dict:
"""批量处理URL"""
results = {
'unique': [],
'duplicate': [],
'stats': {}
}
for url in urls:
self.stats['total_urls'] += 1
if self.is_duplicate(url):
results['duplicate'].append(url)
self.stats['duplicate_urls'] += 1
else:
results['unique'].append(url)
# 更新统计信息
results['stats'] = {
'total_urls': self.stats['total_urls'],
'unique_urls': self.stats['unique_urls'],
'duplicate_urls': self.stats['duplicate_urls'],
'duplicate_rate': self.stats['duplicate_urls'] / self.stats['total_urls'],
'bloom_filter_stats': self.bloom_filter.get_statistics()
}
# 估计误判数量
total_checks = self.stats['total_urls']
estimated_fpr = self.bloom_filter.overall_false_positive_rate()
estimated_fp = total_checks * estimated_fpr
results['stats']['estimated_false_positives'] = estimated_fp
results['stats']['estimated_fpr'] = estimated_fpr
return results
def get_performance_stats(self) -> dict:
"""获取性能统计"""
bloom_stats = self.bloom_filter.get_statistics()
return {
'url_processing': self.stats.copy(),
'bloom_filter': bloom_stats,
'memory_usage_bits': self.bloom_filter.memory_usage(),
'memory_usage_mb': self.bloom_filter.memory_usage() / (8 * 1024 * 1024),
'efficiency': self.stats['unique_urls'] / self.capacity
}7.2 分布式系统中的成员查询
python
复制
下载
class DistributedMembershipService:
"""
分布式系统中的成员查询服务
使用布隆过滤器进行高效成员资格检查
"""
def __init__(self, node_id: str, expected_members: int = 10000):
self.node_id = node_id
self.expected_members = expected_members
# 本地布隆过滤器
self.local_bloom = BloomFilter(expected_members, 0.001)
# 节点间的布隆过滤器同步
self.peer_filters = {} # node_id -> BloomFilter
# 同步策略
self.sync_interval = 60 # 秒
self.last_sync = time.time()
# 统计信息
self.stats = {
'local_members': 0,
'remote_queries': 0,
'false_positives': 0,
'sync_operations': 0
}
def add_local_member(self, member_id: str):
"""添加本地成员"""
self.local_bloom.add(member_id)
self.stats['local_members'] += 1
def query_member(self, member_id: str, check_remote: bool = True) -> dict:
"""
查询成员是否存在
返回包含详细信息的字典
"""
result = {
'member_id': member_id,
'local_check': None,
'remote_checks': {},
'exists': False,
'confidence': 0.0
}
# 本地检查
local_exists = self.local_bloom.contains(member_id)
result['local_check'] = {
'exists': local_exists,
'fpr': self.local_bloom.false_positive_rate()
}
if local_exists:
result['exists'] = True
result['confidence'] = 1.0 - self.local_bloom.false_positive_rate()
elif check_remote:
# 检查远程节点
remote_results = self._query_remote_nodes(member_id)
result['remote_checks'] = remote_results
# 合并结果
if remote_results:
any_remote_exists = any(r['exists'] for r in remote_results.values())
if any_remote_exists:
result['exists'] = True
# 计算综合置信度
min_fpr = min(r['fpr'] for r in remote_results.values()
if r['exists'])
result['confidence'] = 1.0 - min_fpr
# 更新统计
self.stats['remote_queries'] += 1
if result['exists'] and not self._verify_member_exists(member_id):
self.stats['false_positives'] += 1
return result
def _query_remote_nodes(self, member_id: str) -> dict:
"""查询远程节点的布隆过滤器"""
results = {}
for peer_id, peer_filter in self.peer_filters.items():
exists = peer_filter.contains(member_id)
results[peer_id] = {
'exists': exists,
'fpr': peer_filter.false_positive_rate(),
'node_id': peer_id
}
return results
def _verify_member_exists(self, member_id: str) -> bool:
"""
验证成员是否真实存在
这个方法应该在实际系统中实现与数据库的检查
"""
# 这里简化为总是返回True
# 实际系统中应该查询数据库或其他可靠存储
return True
def sync_with_peer(self, peer_id: str, peer_filter_data: dict):
"""
与对等节点同步布隆过滤器
使用布隆过滤器合并策略
"""
if peer_id not in self.peer_filters:
# 创建新的布隆过滤器
self.peer_filters[peer_id] = BloomFilter(
self.expected_members,
0.001
)
# 合并过滤器（按位或操作）
self._merge_filters(self.peer_filters[peer_id], peer_filter_data)
self.stats['sync_operations'] += 1
def _merge_filters(self, local_filter, remote_data):
"""
合并布隆过滤器
假设远程数据包含位数组信息
"""
# 简化实现：实际应处理位数组合并
pass
def compress_for_transmission(self) -> dict:
"""压缩布隆过滤器以便网络传输"""
# 获取本地过滤器的统计信息和位数组
stats = self.local_bloom.get_statistics()
# 创建可传输的数据结构
transmission_data = {
'node_id': self.node_id,
'timestamp': time.time(),
'member_count': self.stats['local_members'],
'filter_stats': stats,
'bit_array': self.local_bloom.bit_array, # 实际应压缩
'hash_functions': self.local_bloom.k,
'bit_array_size': self.local_bloom.m
}
return transmission_data
def get_service_stats(self) -> dict:
"""获取服务统计信息"""
# 计算各种指标
total_queries = self.stats['remote_queries']
false_positive_rate = 0
if total_queries > 0:
false_positive_rate = self.stats['false_positives'] / total_queries
# 内存使用估算
local_memory = self.local_bloom.m # 位
remote_memory = sum(bf.m for bf in self.peer_filters.values())
total_memory = local_memory + remote_memory
return {
'node_id': self.node_id,
'local_members': self.stats['local_members'],
'peer_count': len(self.peer_filters),
'total_queries': total_queries,
'false_positives': self.stats['false_positives'],
'estimated_fpr': false_positive_rate,
'sync_operations': self.stats['sync_operations'],
'memory_usage_bits': total_memory,
'memory_usage_kb': total_memory / (8 * 1024),
'local_filter_fpr': self.local_bloom.false_positive_rate(),
'peer_filter_fprs': {
peer_id: bf.false_positive_rate()
for peer_id, bf in self.peer_filters.items()
}
}篇幅限制下面就只能给大家展示小册部分内容了。整理了一份核心面试笔记包括了：Java面试、Spring、JVM、MyBatis、Redis、MySQL、并发编程、微服务、Linux、Springboot、SpringCloud、MQ、Kafc
需要全套面试笔记及答案
【点击此处即可/免费获取】
7.3 数据库查询优化
python
复制
下载
class DatabaseQueryOptimizer:
"""
使用布隆过滤器优化数据库查询
避免不必要的数据库访问
"""
def __init__(self, db_connection, expected_unique_queries: int = 100000):
"""
初始化查询优化器
Args:
db_connection: 数据库连接
expected_unique_queries: 预期唯一查询数量
"""
self.db = db_connection
# 使用可扩展布隆过滤器
self.query_cache = ScalableBloomFilter(
initial_capacity=expected_unique_queries // 10,
error_rate=0.001, # 低误判率，避免漏查
growth_factor=2.0
)
# 查询结果缓存（可选）
self.result_cache = {} # 查询哈希 -> 结果
# 统计信息
self.stats = {
'total_queries': 0,
'cache_hits': 0,
'db_queries': 0,
'false_positives': 0,
'query_time_saved': 0.0
}
def _hash_query(self, query: str, params: tuple = None) -> str:
"""生成查询的哈希键"""
import hashlib
# 组合查询和参数
if params:
query_str = query + str(params)
else:
query_str = query
# 使用SHA256生成哈希
return hashlib.sha256(query_str.encode()).hexdigest()
def execute_query(self, query: str, params: tuple = None,
use_cache: bool = True) -> list:
"""
执行查询，使用布隆过滤器优化
"""
start_time = time.time()
self.stats['total_queries'] += 1
if not use_cache:
# 直接执行数据库查询
result = self._execute_db_query(query, params)
query_time = time.time() - start_time
return result
# 生成查询哈希
query_hash = self._hash_query(query, params)
# 检查布隆过滤器
if self.query_cache.contains(query_hash):
# 可能已执行过此查询
self.stats['cache_hits'] += 1
# 检查结果缓存
if query_hash in self.result_cache:
# 缓存命中，直接返回结果
result = self.result_cache[query_hash]
query_time = time.time() - start_time
self.stats['query_time_saved'] += query_time
return result
else:
# 布隆过滤器误判（假阳性）
self.stats['false_positives'] += 1
# 继续执行数据库查询
result = self._execute_db_query(query, params)
# 更新缓存
self.result_cache[query_hash] = result
return result
else:
# 新查询，执行数据库查询
result = self._execute_db_query(query, params)
# 添加到布隆过滤器
self.query_cache.add(query_hash)
# 可选：缓存结果
if len(self.result_cache) < 10000: # 限制缓存大小
self.result_cache[query_hash] = result
return result
def _execute_db_query(self, query: str, params: tuple = None) -> list:
"""执行实际的数据库查询"""
self.stats['db_queries'] += 1
# 这里应该使用实际的数据库连接执行查询
# 示例：使用cursor.execute()
try:
# 模拟数据库查询延迟
time.sleep(0.001) # 1ms延迟
# 这里返回模拟结果
return [{"id": 1, "data": "example"}]
except Exception as e:
print(f"数据库查询失败: {e}")
return []
def batch_execute_queries(self, queries: list) -> dict:
"""批量执行查询"""
results = {
'successful': [],
'cached': [],
'new': [],
'stats': {}
}
batch_start = time.time()
for query_info in queries:
query = query_info.get('query')
params = query_info.get('params')
result = self.execute_query(query, params, use_cache=True)
# 分类结果
result_info = {
'query': query[:50] + "..." if len(query) > 50 else query,
'result_count': len(result),
'execution_time': 0.001 # 模拟
}
if self.stats['cache_hits'] > 0:
results['cached'].append(result_info)
else:
results['new'].append(result_info)
results['successful'].append(result_info)
batch_time = time.time() - batch_start
# 更新统计
results['stats'] = {
'total_queries': len(queries),
'batch_execution_time': batch_time,
'avg_query_time': batch_time / len(queries),
'cache_hit_rate': self.stats['cache_hits'] / self.stats['total_queries'],
'false_positive_rate': self.stats['false_positives'] / max(1, self.stats['cache_hits']),
'db_queries_saved': self.stats['cache_hits'] - self.stats['false_positives'],
'estimated_time_saved': self.stats['query_time_saved']
}
return results
def get_optimization_stats(self) -> dict:
"""获取优化统计信息"""
total_queries = self.stats['total_queries']
if total_queries == 0:
cache_hit_rate = 0
false_positive_rate = 0
else:
cache_hit_rate = self.stats['cache_hits'] / total_queries
false_positive_rate = self.stats['false_positives'] / max(1, self.stats['cache_hits'])
# 布隆过滤器统计
bloom_stats = self.query_cache.get_statistics()
return {
'query_statistics': {
'total_queries': total_queries,
'cache_hits': self.stats['cache_hits'],
'db_queries': self.stats['db_queries'],
'false_positives': self.stats['false_positives'],
'cache_hit_rate': cache_hit_rate,
'false_positive_rate': false_positive_rate,
'query_time_saved_seconds': self.stats['query_time_saved']
},
'bloom_filter_statistics': bloom_stats,
'result_cache_size': len(self.result_cache),
'memory_usage_mb': self.query_cache.memory_usage() / (8 * 1024 * 1024)
}
def clear_cache(self):
"""清空缓存"""
self.query_cache = ScalableBloomFilter(
initial_capacity=10000,
error_rate=0.001,
growth_factor=2.0
)
self.result_cache.clear()
# 重置统计
self.stats = {
'total_queries': 0,
'cache_hits': 0,
'db_queries': 0,
'false_positives': 0,
'query_time_saved': 0.0
}八、总结与最佳实践
8.1 关键选择指南
python
复制
下载
class BloomFilterSelectionGuide:
"""布隆过滤器选择指南"""
@staticmethod
def select_filter_type(requirements: dict) -> str:
"""
根据需求选择合适的布隆过滤器类型
Args:
requirements: 包含需求参数的字典
- data_size: 数据量大小（已知/未知）
- need_deletion: 是否需要删除操作
- memory_constraint: 内存限制
- query_performance: 查询性能要求
- false_positive_rate: 目标误判率
- update_frequency: 更新频率
Returns:
推荐的布隆过滤器类型
"""
data_size = requirements.get('data_size', 'known')
need_deletion = requirements.get('need_deletion', False)
memory_constraint = requirements.get('memory_constraint', 'moderate')
query_perf = requirements.get('query_performance', 'moderate')
fpr_target = requirements.get('false_positive_rate', 0.01)
update_freq = requirements.get('update_frequency', 'low')
recommendations = []
# 数据量已知的场景
if data_size == 'known':
if need_deletion:
recommendations.append({
'type': 'CountingBloomFilter',
'counter_size': 4,
'reason': '支持删除操作，counter_size=4提供良好的平衡'
})
else:
if memory_constraint == 'strict':
recommendations.append({
'type': 'StandardBloomFilter',
'reason': '内存效率最高，适合严格内存限制'
})
elif query_perf == 'high':
recommendations.append({
'type': 'BlockedBloomFilter',
'block_size': 256,
'reason': '分块设计提高缓存局部性，查询性能最优'
})
else:
recommendations.append({
'type': 'StandardBloomFilter',
'reason': '通用场景最佳选择'
})
# 数据量未知或增长的场景
else:
if update_freq == 'high':
recommendations.append({
'type': 'ScalableBloomFilter',
'growth_factor': 2.0,
'reason': '自动扩展，避免重建，适合频繁更新的场景'
})
else:
recommendations.append({
'type': 'AdaptiveBloomFilter',
'reason': '动态调整参数，平衡内存和性能'
})
# 特殊场景
if requirements.get('need_compression', False):
recommendations.append({
'type': 'CompressedBloomFilter',
'compression_level': 6,
'reason': '适合网络传输或存储场景'
})
return recommendations
@staticmethod
def calculate_optimal_parameters(n: int, p: float) -> dict:
"""计算最优参数"""
import math
# 计算最优m和k
m = - (n * math.log(p)) / (math.log(2) ** 2)
m = int(math.ceil(m))
k = (m / n) * math.log(2)
k = int(math.ceil(k))
# 计算bits per element
bits_per_element = m / n
return {
'optimal_bit_array_size': m,
'optimal_hash_functions': k,
'bits_per_element': bits_per_element,
'theoretical_fpr': p,
'expected_elements': n,
'estimated_memory_kb': m / (8 * 1024)
}
@staticmethod
def hash_function_recommendations(k: int) -> dict:
"""哈希函数选择推荐"""
recommendations = {
'low_k': {
'range': 'k <= 3',
'recommendation': '使用独立的哈希函数（如MD5、SHA256组合）',
'example': 'hash_funcs = [hash_md5, hash_sha256, hash_murmur]'
},
'medium_k': {
'range': '4 <= k <= 10',
'recommendation': '使用双重哈希策略',
'example': 'gi(x) = h1(x) + i * h2(x) mod m'
},
'high_k': {
'range': 'k > 10',
'recommendation': '使用增强型双重哈希策略',
'example': 'gi(x) = (a*h1(x) + b*h2(x) + c*i) mod m'
}
}
if k <= 3:
return recommendations['low_k']
elif k <= 10:
return recommendations['medium_k']
else:
return recommendations['high_k']8.2 性能优化检查清单
python
复制
下载
class PerformanceOptimizationChecklist:
"""布隆过滤器性能优化检查清单"""
def __init__(self):
self.checklist = {
'参数优化': [
'是否计算了最优的m和k值？',
'bits per element是否在合理范围（通常8-16）？',
'哈希函数数量是否避免过多（通常3-10个）？'
],
'哈希函数选择': [
'是否选择了合适的哈希函数策略？',
'哈希函数是否具有良好的分布特性？',
'是否测试了哈希函数的碰撞率？',
'是否考虑了哈希函数的计算性能？'
],
'内存优化': [
'是否使用了合适的数据结构存储位数组？',
'是否考虑了内存对齐和缓存行大小？',
'对于大过滤器，是否考虑分块存储？',
'是否评估了压缩的可能性？'
],
'误判率管理': [
'是否定期监控实际误判率？',
'是否有误判率超出阈值时的处理策略？',
'是否考虑了误判对业务的影响？',
'是否设置了适当的误判率目标？'
],
'扩展性': [
'是否考虑了数据量增长的情况？',
'是否支持动态扩展？',
'是否有数据迁移策略？',
'是否考虑了分布式场景？'
]
}
def run_optimization_check(self, bloom_filter, config: dict) -> dict:
"""运行优化检查"""
results = {}
for category, checks in self.checklist.items():
results[category] = {}
for check in checks:
# 执行检查
status, details = self._perform_check(check, bloom_filter, config)
results[category][check] = {
'status': status,
'details': details
}
return results
def _perform_check(self, check: str, bloom_filter, config: dict) -> tuple:
"""执行单个检查"""
# 参数优化检查
if '最优的m和k值' in check:
if hasattr(bloom_filter, 'm') and hasattr(bloom_filter, 'k'):
n = getattr(bloom_filter, 'size', 0)
p = config.get('target_fpr', 0.01)
optimal = BloomFilterSelectionGuide.calculate_optimal_parameters(n, p)
current_m = bloom_filter.m
optimal_m = optimal['optimal_bit_array_size']
if abs(current_m - optimal_m) / optimal_m < 0.1: # 10%误差范围内
return 'PASS', f'当前m={current_m}, 最优m={optimal_m}'
else:
return 'WARN', f'建议调整: 当前m={current_m}, 最优m={optimal_m}'
# 哈希函数检查
elif '哈希函数策略' in check:
if hasattr(bloom_filter, 'hash_funcs'):
k = len(bloom_filter.hash_funcs)
recommendation = BloomFilterSelectionGuide.hash_function_recommendations(k)
return 'INFO', f'当前k={k}, 推荐: {recommendation["recommendation"]}'
# 内存优化检查
elif '数据结构存储' in check:
if hasattr(bloom_filter, 'bit_array'):
# 检查是否使用高效的数据结构
bit_array_type = type(bloom_filter.bit_array).__name__
if bit_array_type in ['list', 'array', 'bytearray']:
return 'PASS', f'使用{bit_array_type}存储位数组'
else:
return 'WARN', f'考虑使用更高效的数据结构'
# 误判率检查
elif '监控实际误判率' in check:
if hasattr(bloom_filter, 'false_positive_rate'):
fpr = bloom_filter.false_positive_rate()
target_fpr = config.get('target_fpr', 0.01)
if fpr <= target_fpr * 1.5: # 允许50%的误差
return 'PASS', f'当前FPR={fpr:.6f}, 目标={target_fpr}'
else:
return 'FAIL', f'FPR超标: 当前={fpr:.6f}, 目标={target_fpr}'
return 'UNKNOWN', '未执行检查'
def generate_optimization_report(self, check_results: dict) -> str:
"""生成优化报告"""
report_lines = []
report_lines.append("=" * 80)
report_lines.append("布隆过滤器性能优化报告")
report_lines.append("=" * 80)
total_checks = 0
passed_checks = 0
failed_checks = 0
for category, checks in check_results.items():
report_lines.append(f"\n{category}:")
for check, result in checks.items():
total_checks += 1
status = result['status']
details = result['details']
if status == 'PASS':
passed_checks += 1
symbol = '✓'
elif status == 'FAIL':
failed_checks += 1
symbol = '✗'
elif status == 'WARN':
symbol = '⚠'
else:
symbol = '?'
report_lines.append(f" {symbol} {check}")
if details:
report_lines.append(f" {details}")
# 总结
report_lines.append("\n" + "=" * 80)
report_lines.append("优化建议总结:")
report_lines.append("=" * 80)
warning_count = total_checks - passed_checks - failed_checks
report_lines.append(f"总检查数: {total_checks}")
report_lines.append(f"通过: {passed_checks}")
report_lines.append(f"警告: {warning_count}")
report_lines.append(f"失败: {failed_checks}")
if failed_checks > 0:
report_lines.append("\n⚠ 存在关键问题需要解决!")
return "\n".join(report_lines)这个全面的布隆过滤器指南涵盖了从基础理论到高级优化的所有方面，包括误判率分析、哈希函数选择、各种变种实现以及实际应用案例。实际使用时，应根据具体场景和需求选择最合适的实现。