scikit-learn学习Day30

1.数据集划分

import numpy as np
from sklearn.datasets import load_iris
import pandas as pd
arr = load_iris()
data = arr.data
target = arr.target
print(target)
n_target = target.reshape(len(target),1)
print(n_target)
n_data = np.hstack([data,n_target])
print(n_data)
columns = arr.feature_names
print(columns)
columns.append('Target')
print(columns)
Data = pd.DataFrame(n_data,columns=columns)
print(Data)


from sklearn.datasets import fetch_20newsgroups
# data_home：数据集存储位置，subset：'train','test','all'分别代表训练/测试/全部数据集
data = fetch_20newsgroups(data_home='src')


# 打开excel和csv文件
import pandas as pd
data = pd.read_csv('src/ss.csv')
data1 = pd.read_excel('src/test.xlsx')
print(data,'\n',data1)
#
# 数据集的划分
#
import pandas as pd
from sklearn.datasets import load_iris
from sklearn.model_selection import train_test_split
import numpy as np
arr = load_iris()
data = arr.data
target = arr.target
# *arrays：特征或目标，train_size/test_size：划分比例（0-1或整数），random_state：固定随机数种子，stratify：分层，保证train和test中包括数据库中所有的类别样本
x_train, x_test, y_train, y_test = train_test_split(data,target,train_size=0.8,stratify=target,random_state=6)
print(x_train)
print(y_test)

# 二维数组划分只改变第一维度，不改变第二维度
arr1 = np.arange(1,16,1).reshape(5,3)
np_train,np_test = train_test_split(arr1,train_size=0.8)
print(np_train)

# DataFrame划分后还是DataFrame
arr1 = np.arange(1,16,1).reshape(5,3)
data = pd.DataFrame(arr1)
pd_train,pd_test = train_test_split(data,train_size=0.8)
print(pd_train)



from sklearn.feature_extraction import DictVectorizer
data = [{'city':'成都', 'age':30, 'temperature':20},
        {'city':'重庆','age':33, 'temperature':60},
        {'city':'北京', 'age':42, 'temperature':80},
        {'city':'上海', 'age':22, 'temperature':70},
        {'city':'成都', 'age':72, 'temperature':40},
       ]
# 直接对字典划分得到的数据集还是原格式
x_train,x_test = train_test_split(data,train_size=0.8)
print(x_train)
# 使用DictVectorizer提取特征
transfer = DictVectorizer(sparse=True)#sparse：是否将数据集转换成三元组形式
data_new = transfer.fit_transform(data)
print("data_new:\n", data_new)
x = data_new.toarray()#将三元组转为ndarray
print(type(x))
print(x)
x_train,x_test = train_test_split(x,train_size=0.8,random_state=6)
print(x_train)

2.特征提取

# 字典列表特征提取
from sklearn.feature_extraction import DictVectorizer
import pandas as pd
data = [{'city':'北京','money':38,'age':20},
       {'city':'上海','money':60,'age':29},
       {'city':'深圳','money':30,'age':32},
       {'city':'深圳','money':40,'age':49},
       ]
# 初始化工具(字典变成向量的工具器)
model = DictVectorizer(sparse=False)#sparse是否转换成三元组形式
data = model.fit_transform(data)
print(data,type(data))
print(model.get_feature_names_out())#获取所有的特征名称
ddata = pd.DataFrame(data=data,columns=model.get_feature_names_out())
# print(ddata)
ddata

# 字典列表特征提取
from sklearn.feature_extraction import DictVectorizer
import pandas as pd
data = [{'city':'北京','money':38,'age':20},
       {'city':'上海','money':60,'age':29},
       {'city':'深圳','money':30,'age':32},
       {'city':'深圳','money':40,'age':49},
       ]
# 初始化工具(字典变成向量的工具器)
model = DictVectorizer(sparse=True)#sparse是否转换成三元组形式
data = model.fit_transform(data)
print(data,type(data))
arr = data.toarray()#把三元组（稀疏矩阵）转化为数组
print(arr)

# 文本特征提取
from sklearn.feature_extraction.text import CountVectorizer
corpus = ['I love machine learning. Its awesome.', 'Its a book amazon book.','Amazon is book great company']
# 创建一个词频提取对象
vectorizer = CountVectorizer(stop_words=['amazon'])
# 提取词频
x = vectorizer.fit_transform(corpus)
print(x)
print(x.toarray())
print(vectorizer.get_feature_names_out())

# 中文文本特征提取
from sklearn.feature_extraction.text import CountVectorizer
import jieba
import pandas as pd
# arr = list(jieba.cut('我爱北京天安门'))
# print(arr)
# str1 = ' '.join(arr)
# print(str1)
#传入的文本（没有断词的字符串）用jieba分词工具转化为数据容器，然后再把数据容器中的元素连接成字符串
def mycut(text):
    return ' '.join(jieba.cut(text))

corpus = ['我爱北京天安门','我爱成都天府']
# # 创建一个词频提取对象
vectorizer = CountVectorizer(stop_words=[])
# # 提取词频
data = [mycut(el) for el in corpus]
print(data)
x = vectorizer.fit_transform(data)
print(x)
print(x.toarray())
print(vectorizer.get_feature_names_out())
ddata = pd.DataFrame(x.toarray(),columns=vectorizer.get_feature_names_out())
ddata

import jieba
import pandas as pd
from sklearn.feature_extraction.text import TfidfVectorizer

def cut_words(text):
    return " ".join(list(jieba.cut(text)))

data = ["教育学会会长期间，坚定支持民办教育事业！",  "扶持民办,学校发展事业","事业做出重大贡献！"]
data_new = [cut_words(v) for v in data]

transfer = TfidfVectorizer(stop_words=['期间', '做出',"重大贡献"]) 
data_final = transfer.fit_transform(data_new)

pd.DataFrame(data_final.toarray(), columns=transfer.get_feature_names_out())

# TfidfVectorizer TF-IDF文本特征词的重要程度特征提取
from sklearn.feature_extraction.text import CountVectorizer,TfidfVectorizer
import jieba
import pandas as pd
def mycut(text):
    return ' '.join(jieba.cut(text))
data = ["教育学会会长期间，坚定支持民办教育事业！",  "扶持民办,学校发展事业","事业做出重大贡献！"]
data = [mycut(i) for i in data]
# print(data)
transfer = TfidfVectorizer(stop_words=[])
res = transfer.fit_transform(data)
print(transfer.get_feature_names_out())
print(res.toarray())

ddata = pd.DataFrame(res.toarray(),columns=transfer.get_feature_names_out())
ddata

from sklearn.preprocessing import MinMaxScaler
import pandas as pd
scaler = MinMaxScaler(feature_range=(0,1))
data = pd.read_excel('src/test2.xlsx')
print(data.values)
# arr = scaler.fit_transform(data.values)
arr = scaler.fit_transform(data)
print(arr)

# 字典列表特征提取
from sklearn.feature_extraction import DictVectorizer
from sklearn.preprocessing import MinMaxScaler
import pandas as pd
data = [{'city':'北京','money':38,'age':20},
       {'city':'上海','money':60,'age':29},
       {'city':'深圳','money':30,'age':32},
       {'city':'深圳','money':40,'age':49},
       ]
# 初始化工具(字典变成向量的工具器)
model = DictVectorizer(sparse=True)#sparse是否转换成三元组形式
data = model.fit_transform(data)
print(data,type(data))
arr = data.toarray()#把三元组（稀疏矩阵）转化为数组
print(arr)
arr1 = MinMaxScaler().fit_transform(arr)
arr1

from sklearn.preprocessing import StandardScaler
import numpy as np

# 初始化标准化工具
scaler = StandardScaler()
np.random.seed(7)
data = np.random.randint(0,100,(30,4))
# print(data)
x = scaler.fit_transform(data)
print(x)

# fit 和 transfer 和 fit_transform的区别
from sklearn.preprocessing import StandardScaler,MinMaxScaler
import numpy as np
np.random.seed(666)
transfer = StandardScaler()
x = np.random.randint(100,size=(3,4))
print(x)
# y = transfer.fit_transform(x)
transfer.fit(x)#计算均值和标准差
x = transfer.transform(x)
print(x)
x2 = np.array([[10,20,30,40]])
y = transfer.transform(x2)
print(y)

# 低方差过滤
from sklearn.feature_selection import VarianceThreshold
transer = VarianceThreshold(threshold=0.01)
x = [
    [0,2,0,3],
    [0,1,4,3],
    [0,1,1,3]
]
x = transer.fit_transform(x)
print(x)

from scipy.stats import pearsonr
import pandas as pd
data = pd.read_csv('src/factor_returns.csv')
print(data)
data = data.iloc[:,1:-2]
print(data)
r1 = pearsonr(data["pe_ratio"], data["pb_ratio"])
print(r1.statistic) #-0.0043893227799362555 相关性, 负数表示负相关
print(r1.pvalue) #0.8327205496590723        正相关性
r2 = pearsonr(data['revenue'], data['total_expense'])
print(r2) #PearsonRResult(statistic=0.9958450413136111, pvalue=0.0)