import numpy as np
sigmoid = lambda x:1/(1 + np.exp(-x))
relu = lambda x:(x>0).astype(float)*x
weights = np.array([[1,4],[4,1]])
activation = sigmoid(np.array([1,0.01]))
print("Activations")
activations = list()
for iter in range(10):
activation = sigmoid(activation.dot(weights))
activations.append(activation)
print(activation)
print("\nGradients")
gradient = np.ones_like(activation)
for activation in reversed(activations):
gradient = (activation * (1 - activation) * gradient)
gradient = gradient.dot(weights.transpose())
print(gradient)
print("Relu Activations")
activations = list()
for iter in range(10):
activation = relu(activation.dot(weights))
activations.append(activation)
print(activation)
print("\nRelu Gradients")
gradient = np.ones_like(activation)
for activation in reversed(activations):
gradient = ((activation > 0) * gradient).dot(weights.transpose())
print(gradient)
'''
Activations
[0.93940638 0.96852968]
[0.9919462 0.99121735]
[0.99301385 0.99302901]
[0.9930713 0.99307098]
[0.99307285 0.99307285]
[0.99307291 0.99307291]
[0.99307291 0.99307291]
[0.99307291 0.99307291]
[0.99307291 0.99307291]
[0.99307291 0.99307291]
Gradients
[0.03439552 0.03439552]
[0.00118305 0.00118305]
[4.06916726e-05 4.06916726e-05]
[1.39961115e-06 1.39961115e-06]
[4.81403643e-08 4.81403637e-08]
[1.65582672e-09 1.65582765e-09]
[5.69682675e-11 5.69667160e-11]
[1.97259346e-12 1.97517920e-12]
[8.45387597e-14 8.02306381e-14]
[1.45938177e-14 2.16938983e-14]
Relu Activations
[4.8135251 4.72615519]
[23.71814585 23.98025559]
[119.63916823 118.852839 ]
[595.05052421 597.40951192]
[2984.68857188 2977.61160877]
[14895.13500696 14916.36589628]
[74560.59859209 74496.90592414]
[372548.22228863 372739.30029248]
[1863505.42345854 1862932.18944699]
[9315234.18124649 9316953.88328115]
Relu Gradients
[5. 5.]
[25. 25.]
[125. 125.]
[625. 625.]
[3125. 3125.]
[15625. 15625.]
[78125. 78125.]
[390625. 390625.]
[1953125. 1953125.]
[9765625. 9765625.]
'''
54、深度学习-自学之路-自己搭建深度学习框架-15、解释梯度消失和梯度爆炸的问题。
小宇爱2025-03-03 2:02
相关推荐
狂师几秒前
啥是AI Agent!2025年值得推荐入坑AI Agent的五大工具框架!(新手科普篇)星辰大海的精灵3 分钟前
使用Docker和Kubernetes部署机器学习模型victory04315 分钟前
SpiceMix enables integrative single-cell spatial modeling of cell identity 文章解读新智元9 分钟前
半数清华,8 位华人 AI 天团集体投奔 Meta!奥特曼:砸钱抢人不如培养死忠新智元12 分钟前
全球顶尖 CS 论文惊爆 AI「好评密令」!哥大等 14 所高校卷入,学术圈炸锅l0sgAi17 分钟前
vLLM在RTX50系显卡上部署大模型-使用wsl2DDliu17 分钟前
花半个月死磕提示词后,我发现:真正值钱的不是模板,是这套可复用的结构化思维腾讯云开发者17 分钟前
AI 浪潮下的锚与帆:工程师文化的变与不变 | 架构师夜生活JoernLee18 分钟前
机器学习算法:支持向量机SVM杰尼橙子23 分钟前
深度解读Karpathy说的Software 3.0时代,感觉是个人的机会很大的时代呀