LSTM语言模型验证代码

#任务:基于已有文本数据,建立LSTM模型,预测序列文字

1 完成数据预处理,将文字序列数据转化为可用于LSTM输入的数据。

2 查看文字数据预处理后的数据结构,并进行数据分离操作

3 针对字符串输入("In the heart of the ancient, dense forest, where sunlight filtered through the thick canopy in a patchwork of gold and shadow, a creature of unparalleled grace and mystery roamed. This was no ordinary animal; it was a panther, a living embodiment of the wild's s untamed beauty and raw power. Its presence sent shivers down the spines of every creature that shared its domain, a silent yet potent reminder of nature's hierarchy.")预测其后续对应的字符

备注:模型结构:单层LSTM,输出20个神经元;每次使用前20个字符预测第21个字符。

#l载入数据

data = open('test.txt').read()

#移除换行符

data = data.replace('\n','').replace('\r','')

print(data)

#字符去重

letters = list(set(data))

print(letters)

num_letters = len(letters)

print(num_letters)

#建立字典,字母到数字的映射关系

int_to_char={a:b for a,b in enumerate(letters)}

print(int_to_char)

char_to_int={b:a for a,b in enumerate(letters)}

print(char_to_int)

----------生成训练数据---------

time_step = 20

import numpy as np

from keras.utils import to_categorical

滑动窗口提取数据

def extract_data(data, slide):

x = []

y = []

for i in range(len(data) - slide):

x.append([a for a in data[i:i+slide]])

y.append(data[i+slide]) # 修正变量名,将side改为slide

return x, y

字符到数字的批量转化

def char_to_int_Data(x, y, char_to_int):

x_to_int = []

y_to_int = []

for i in range(len(x)):

x_to_int.append([char_to_int[char] for char in x[i]])

y_to_int.append([char_to_int[char] for char in y[i]])

return x_to_int, y_to_int

实现输入字符文章的批量处理,输入整个字符,滑动窗大小,转化字典

def data_preprocessing(data, slide, num_letters, char_to_int):

char_Data = extract_data(data, slide)

int_Data = char_to_int_Data(char_Data[0], char_Data[1], char_to_int)

Input = int_Data[0]

Output = list(np.array(int_Data[1]).flatten())

Input_RESHAPED = np.array(Input).reshape(len(Input), slide)

创建全零数组,然后用独热编码填充

one_hot_input = np.zeros((Input_RESHAPED.shape[0], Input_RESHAPED.shape[1], num_letters))

修正嵌套循环的索引

for i in range(Input_RESHAPED.shape[0]): # 遍历样本

for j in range(Input_RESHAPED.shape[1]): # 遍历时间步

one_hot_input[i, j, :] = to_categorical(Input_RESHAPED[i, j], num_classes=num_letters)

return one_hot_input, Output

#提取X和y的数据

X, y = data_preprocessing(data, time_step, num_letters, char_to_int)

#分离数据

from sklearn.model_selection import train_test_split

X_train, X_test, y_train, y_test = train_test_split(X,y, test_size=0.1,random_state=10)

print(X_train.shape, len(y_train))

y_train_category = to_categorical(y_train, num_letters)

print(y_train_category)

#setup the model

from keras.models import Sequential

from keras.layers import Dense, LSTM

model =Sequential()

model.add(LSTM(units=20, input_shape=(X_train.shape[1], X_train.shape[2]), activation='relu'))

model.add(Dense(units=num_letters, activation='softmax'))

model.compile(optimizer='adam', loss = 'categorical_crossentropy', metrics=['accuracy'])

model.summary()

#train the model

model.fit(X_train,y_train_category,batch_size=1000,epochs=50)

#make prediction based on the training data

假设这是一个分类模型(如多类或二分类)

使用 model.predict() 替代 predict_classes()

y_train_predict_probs = model.predict(X_train)

对于多分类问题(softmax输出),获取预测类别索引

if y_train_predict_probs.shape[1] > 1:

y_train_predict = np.argmax(y_train_predict_probs, axis=1)

对于二分类问题(sigmoid输出),使用阈值0.5

else:

y_train_predict = (y_train_predict_probs > 0.5).astype(int)

print(y_train_predict)

#transform the int to letters

y_train_predict_char = [int_to_char[i] for i in y_train_predict]

print(y_train_predict_char)

from sklearn.metrics import accuracy_score

accuracy_train = accuracy_score(y_train, y_train_predict)

print(accuracy_train)

#make prediction based on the training data

假设这是一个分类模型(如多类或二分类)

使用 model.predict() 替代 predict_classes()

y_test_predict_probs = model.predict(X_test)

对于多分类问题(softmax输出),获取预测类别索引

if y_test_predict_probs.shape[1] > 1:

y_test_predict = np.argmax(y_test_predict_probs, axis=1)

对于二分类问题(sigmoid输出),使用阈值0.5

else:

y_test_predict = (y_test_predict_probs > 0.5).astype(int)

#transform the int to letters

y_test_predict_char = [int_to_char[i] for i in y_test_predict]

print(y_test_predict_char)

from sklearn.metrics import accuracy_score

accuracy_test = accuracy_score(y_test, y_test_predict)

print(accuracy_test)

print(y_test_predict)

print(y_test)

new_letters = 'In the heart of the ancient, dense forest, where sunlight filtered through the thick canopy in a patchwork of gold and shadow, a creature of unparalleled grace and mystery roamed. '

#new_letters = 'fifin is a student in sf industry. He studiess her her fo sssssssss'

X_new,y_new = data_preprocessing(new_letters, time_step, num_letters, char_to_int)

#make prediction based on the training data

假设这是一个分类模型(如多类或二分类)

使用 model.predict() 替代 predict_classes()

y_new_predict_probs = model.predict(X_new)

对于多分类问题(softmax输出),获取预测类别索引

if y_new_predict_probs.shape[1] > 1:

y_new_predict = np.argmax(y_new_predict_probs, axis=1)

对于二分类问题(sigmoid输出),使用阈值0.5

else:

y_new_predict = (y_new_predict_probs > 0.5).astype(int)

print(y_new_predict)

#transform the int to letters

y_new_predict_char = [int_to_char[i] for i in y_new_predict]

print(y_new_predict_char)

for i in range(0, X_new.shape[0]-20):

print(new_letters[i:i+20], '--predict next letter is ---', y_new_predict_char[i])

相关推荐
美狐美颜sdk1 小时前
跨平台直播美颜SDK集成实录:Android/iOS如何适配贴纸功能
android·人工智能·ios·架构·音视频·美颜sdk·第三方美颜sdk
DeepSeek-大模型系统教程2 小时前
推荐 7 个本周 yyds 的 GitHub 项目。
人工智能·ai·语言模型·大模型·github·ai大模型·大模型学习
郭庆汝2 小时前
pytorch、torchvision与python版本对应关系
人工智能·pytorch·python
小雷FansUnion4 小时前
深入理解MCP架构:智能服务编排、上下文管理与动态路由实战
人工智能·架构·大模型·mcp
资讯分享周4 小时前
扣子空间PPT生产力升级:AI智能生成与多模态创作新时代
人工智能·powerpoint
叶子爱分享5 小时前
计算机视觉与图像处理的关系
图像处理·人工智能·计算机视觉
鱼摆摆拜拜5 小时前
第 3 章:神经网络如何学习
人工智能·神经网络·学习
一只鹿鹿鹿5 小时前
信息化项目验收,软件工程评审和检查表单
大数据·人工智能·后端·智慧城市·软件工程
张较瘦_5 小时前
[论文阅读] 人工智能 | 深度学习系统崩溃恢复新方案:DaiFu框架的原位修复技术
论文阅读·人工智能·深度学习
cver1235 小时前
野生动物检测数据集介绍-5,138张图片 野生动物保护监测 智能狩猎相机系统 生态研究与调查
人工智能·pytorch·深度学习·目标检测·计算机视觉·目标跟踪