机器学习-07-分类回归和聚类算法评估函数及案例

from sklearn.model_selection import cross_val_score
 from sklearn.model_selection import GridSearchCV

#导入数据集模块
from sklearn import datasets
#分别加载iris和digits数据集
iris_dataset = datasets.load_iris()  #鸢尾花数据集
# print(dir(datasets))
# print(iris_dataset.keys())
# dict_keys(['data', 'target', 'frame', 'target_names', 'DESCR', 'feature_names', 'filename', 'data_module'])

from sklearn.model_selection import train_test_split
X_train,X_test,y_train,y_test=train_test_split(iris.data,iris.target,test_size=0.4,random_state=0)
print("iris.data[0:5]:\n",iris.data[0:5])
print("iris.target[0:5]:\n",iris.target[0:5])
print("iris.data.shape:",iris.data.shape)
print("iris.target.shape:",iris.target.shape)
print("X_train.shape:",X_train.shape)
print("y_train.shape:",y_train.shape)
print("X_test.shape:",X_test.shape)
print("y_test.shape:",y_test.shape)

# 第二步使用sklearn模型的选择
from sklearn import svm
svc = svm.SVC(gamma='auto')

#第三步使用sklearn模型的训练
svc.fit(X_train, y_train)

# 第四步使用sklearn进行模型预测
print(svc.predict([[5.84,4.4,6.9,2.5]]))

#第五步机器学习评测的指标
#机器学习库sklearn中，我们使用metrics方法实现：
import numpy as np
from sklearn.metrics import accuracy_score
print("y_test:\n",y_test)
y_pred = svc.predict(X_test)
print("y_pred:\n",y_pred)
print(accuracy_score(y_test, y_pred))

#第五步机器学习评测方法：交叉验证 (Cross validation)
#机器学习库sklearn中，我们使用cross_val_score方法实现：
from sklearn.model_selection import cross_val_score
scores = cross_val_score(svc, iris.data, iris.target, cv=5)
print(scores)


#第六步机器学习:模型的保存
#机器学习库sklearn中，我们使用joblib方法实现：
# from sklearn.externals import joblib
import joblib
joblib.dump(svc, 'filename.pkl') 
svc1 = joblib.load('filename.pkl') 
#测试读取后的Model
print(svc1.score(X_test, y_test))

from sklearn.metrics import fbeta_score, make_scorer 
ftwo_scorer = make_scorer(fbeta_score, beta=2) 
from sklearn.model_selection import GridSearchCV 
from sklearn.svm import LinearSVC 
grid = GridSearchCV(LinearSVC(), param_grid={'C': [1, 10]}, scoring=ftwo_scorer)
grid

import numpy as np 

# 自定义评估函数
def my_custom_loss_func(ground_truth, predictions): 
    diff = np.abs(ground_truth - predictions).max() 
    return np.log(1 + diff) 

# 封装自定义的评估函数
from sklearn.metrics import make_scorer 
loss = make_scorer(my_custom_loss_func, greater_is_better=False) 
score = make_scorer(my_custom_loss_func, greater_is_better=True) 
ground_truth = [[1, 1],[2, 2]] 
predictions = [0, 1] 
# 虚拟分类器
from sklearn.dummy import DummyClassifier 
clf = DummyClassifier(strategy='most_frequent', random_state=0) 
clf = clf.fit(ground_truth, predictions) 
# 输出结构
print(loss(clf,ground_truth, predictions)) # -0.6931471805599453
print(score(clf,ground_truth, predictions)) # 0.6931471805599453

### 3.confusion_matrix
from sklearn import datasets
from sklearn.svm import LinearSVC
from sklearn.model_selection import cross_validate
from sklearn.metrics import confusion_matrix,ConfusionMatrixDisplay
import matplotlib.pyplot as plt



y_true = [1, 0, 1, 1, 0, 1]
y_pred = [0, 0, 1, 1, 0, 1]
 
cm=confusion_matrix(y_true, y_pred)
print(confusion_matrix(y_true, y_pred))
print(confusion_matrix(y_true, y_pred,normalize='all'))
tn, fp, fn, tp = confusion_matrix(y_true, y_pred).ravel()
print("tn  fp  fn  tp")
print(tn, fp, fn, tp)
disp = ConfusionMatrixDisplay(confusion_matrix=cm)
disp.plot()
# plt.show()

print("----------------")
y_true = [2, 0, 2, 2, 0, 1]
y_pred = [0, 0, 2, 2, 0, 2]
 
cm=confusion_matrix(y_true, y_pred)
print(confusion_matrix(y_true, y_pred))
print(confusion_matrix(y_true, y_pred,normalize='all'))
disp = ConfusionMatrixDisplay(confusion_matrix=cm)
disp.plot()
# plt.show()

import numpy as np
from sklearn.metrics import accuracy_score
y_pred = [0, 2, 1, 3]
y_true = [0, 1, 2, 3]
print(accuracy_score(y_true, y_pred)) # 0.5
print(accuracy_score(y_true, y_pred, normalize=False)) # 2

from sklearn import metrics 
y_pred = [0, 1, 0, 0] 
y_true = [0, 1, 0, 1] 
print("precision_score",metrics.precision_score(y_true, y_pred)) # 1.0 
print("recall_score",metrics.recall_score(y_true, y_pred)) #0.5 
print("f1_score",metrics.f1_score(y_true, y_pred) )#0.66... 
print("fbeta_score",metrics.fbeta_score(y_true, y_pred, beta=0.5)) #0.83... 
print("fbeta_score",metrics.fbeta_score(y_true, y_pred, beta=1)) #0.66... 
print("fbeta_score",metrics.fbeta_score(y_true, y_pred, beta=2)) #0.55... 
print("precision_recall_fscore_support",metrics.precision_recall_fscore_support(y_true, y_pred, beta=0.5) )
#(array([ 0.66..., 1. ]), array([ 1. , 0.5]), array([ 0.71..., 0.83...]), array([2, 2]...)) 
import numpy as np 
from sklearn.metrics import precision_recall_curve 
from sklearn.metrics import average_precision_score 
y_true = np.array([0, 0, 1, 1]) 
y_scores = np.array([0.1, 0.4, 0.35, 0.8]) 
precision, recall, threshold = precision_recall_curve(y_true, y_scores) 

# from sklearn.metrics import precision_recall_curve
from sklearn.metrics import PrecisionRecallDisplay

pr_display = PrecisionRecallDisplay(precision=precision, recall=recall).plot()

print("precision",precision) # array([ 0.66..., 0.5 , 1. , 1. ]) 
print("recall",recall) #array([ 1. , 0.5, 0.5, 0. ]) 
print("threshold",threshold) #array([ 0.35, 0.4 , 0.8 ]) 
print("average_precision_score",average_precision_score(y_true, y_scores)) #0.79...

from sklearn.metrics import classification_report 
y_true = [0, 1, 2, 2, 0,0, 1, 2, 2, 0] 
y_pred = [0, 0, 2, 2, 0,0, 1, 2, 0, 0] 
target_names = ['class 0', 'class 1', 'class 2'] 
print(classification_report(y_true, y_pred, target_names=target_names))

print("### 7. 回归预测的r2_score ")
# r2_score函数计算决定系数，通常表示为R²。
# 它表示模型中自变量所解释的方差（Y）的比例。它提供了拟合度的指示，
# 因此通过解释方差的比例来衡量未见过的样本可能被模型预测的程度。
 
from sklearn.metrics import r2_score
y_true = [3, -0.5, 2, 7]
y_pred = [2.5, 0.0, 2, 8]
print(r2_score(y_true, y_pred))
 
print("### 8. 回归预测的mean_absolute_error")
from sklearn.metrics import mean_absolute_error
y_true = [3, -0.5, 2, 7]
y_pred = [2.5, 0.0, 2, 8]
print(mean_absolute_error(y_true, y_pred))
 
print("### 9. 回归预测的mean_squared_error ")
from sklearn.metrics import mean_squared_error
y_true = [3, -0.5, 2, 7]
y_pred = [2.5, 0.0, 2, 8]
print(mean_squared_error(y_true, y_pred))
 
print("### 10. 回归预测的mean_squared_log_error")
from sklearn.metrics import mean_squared_log_error
y_true = [3, 5, 2.5, 7]
y_pred = [2.5, 5, 4, 8]
print(mean_squared_log_error(y_true, y_pred))

print("### 11.无监督聚类的Silhouette Coefficient")
from sklearn import metrics
from sklearn import datasets
import numpy as np
X, y = datasets.load_iris(return_X_y=True)
 
from sklearn.cluster import KMeans
kmeans_model = KMeans(n_clusters=3, random_state=1).fit(X)
labels = kmeans_model.labels_
print(metrics.silhouette_score(X, labels, metric='euclidean'))

from sklearn import metrics
# 查看模块的函数
dir(metrics)
 
print("### 1.Accuracy score")
import numpy as np
from sklearn.metrics import accuracy_score
 
y_pred = [0, 2, 1, 3]
y_true = [0, 1, 2, 3]
print(accuracy_score(y_true, y_pred))
print(accuracy_score(y_true, y_pred, normalize=False))  # 正确的预测个数
 
print("### 2.Top-k accuracy score")
# top_k_accuracy_score函数是accuracy_score的泛化。
# 区别在于，只要真实标签与k个最高预测分数之一相关联，预测就被认为是正确的。
# 准确度_分数是k=1的特殊情况。
import numpy as np
from sklearn.metrics import top_k_accuracy_score
y_true = np.array([0, 1, 2, 2])
y_score = np.array([[0.5, 0.2, 0.2],
                    [0.3, 0.4, 0.2],
                    [0.2, 0.4, 0.3],
                    [0.7, 0.2, 0.1]])
print(top_k_accuracy_score(y_true, y_score, k=2))
 
# Not normalizing gives the number of "correctly" classified samples
print(top_k_accuracy_score(y_true, y_score, k=2, normalize=False))
 
print("### 3.confusion_matrix")
from sklearn import datasets
from sklearn.svm import LinearSVC
from sklearn.model_selection import cross_validate
from sklearn.metrics import confusion_matrix,ConfusionMatrixDisplay
import matplotlib.pyplot as plt
 
y_true = [2, 0, 2, 2, 0, 1]
y_pred = [0, 0, 2, 2, 0, 2]
 
cm=confusion_matrix(y_true, y_pred)
print(confusion_matrix(y_true, y_pred))
print(confusion_matrix(y_true, y_pred,normalize='all'))
disp = ConfusionMatrixDisplay(confusion_matrix=cm)
disp.plot()
plt.show()
 
print("# 二分类")
y_true = [0, 0, 0, 1, 1, 1, 1, 1]
y_pred = [0, 1, 0, 1, 0, 1, 0, 1]
tn, fp, fn, tp = confusion_matrix(y_true, y_pred).ravel()
print(tn, fp, fn, tp)
 
print("# A sample toy binary classification dataset")
X, y = datasets.make_classification(n_classes=2, random_state=0)
svm = LinearSVC(random_state=0)
def confusion_matrix_scorer(clf, X, y):
    y_pred = clf.predict(X)
    cm = confusion_matrix(y, y_pred)
    return {'tn': cm[0, 0], 'fp': cm[0, 1],
            'fn': cm[1, 0], 'tp': cm[1, 1]}
cv_results = cross_validate(svm, X, y, cv=5,
                            scoring=confusion_matrix_scorer)
print("# Getting the test set true positive scores")
print(cv_results['test_tp'])
 
# Getting the test set false negative scores
print(cv_results['test_fn'])
 
print(cv_results['test_tn'])
print(cv_results['test_fp'])
 
print("### 4.classification_report")
from sklearn.metrics import classification_report
y_true = [0, 1, 2, 2, 0]
y_pred = [0, 0, 2, 1, 0]
target_names = ['class 0', 'class 1', 'class 2']
print(classification_report(y_true, y_pred, target_names=target_names))
 
print("### 5. hamming_loss")
from sklearn.metrics import hamming_loss
y_pred = [1, 2, 3, 4]
y_true = [2, 2, 3, 4]
print(hamming_loss(y_true, y_pred))
 
print("### 6. Precision, recall and F-measures")
from sklearn import metrics
y_pred = [0, 1, 0, 0]
y_true = [0, 1, 0, 1]
print(metrics.precision_score(y_true, y_pred))
print(metrics.recall_score(y_true, y_pred))
print(metrics.f1_score(y_true, y_pred))
print(metrics.fbeta_score(y_true, y_pred, beta=0.5))
print(metrics.fbeta_score(y_true, y_pred, beta=1))
print(metrics.fbeta_score(y_true, y_pred, beta=2))
print(metrics.precision_recall_fscore_support(y_true, y_pred, beta=0.5))
 
import numpy as np
from sklearn.metrics import precision_recall_curve
from sklearn.metrics import average_precision_score
y_true = np.array([0, 0, 1, 1])
y_scores = np.array([0.1, 0.4, 0.35, 0.8])
precision, recall, threshold = precision_recall_curve(y_true, y_scores)
print(precision)
print(recall)
print(threshold)
print( average_precision_score(y_true, y_scores))
 
print("## 多分类")
from sklearn import metrics
y_true = [0, 1, 2, 0, 1, 2]
y_pred = [0, 2, 1, 0, 0, 1]
print(metrics.precision_score(y_true, y_pred, average='macro'))
print(metrics.recall_score(y_true, y_pred, average='micro'))
print(metrics.f1_score(y_true, y_pred, average='weighted'))
print(metrics.fbeta_score(y_true, y_pred, average='macro', beta=0.5))
print(metrics.precision_recall_fscore_support(y_true, y_pred, beta=0.5, average=None))
 
print("### 7. 回归预测的r2_score ")
# r2_score函数计算决定系数，通常表示为R²。
# 它表示模型中自变量所解释的方差（Y）的比例。它提供了拟合度的指示，
# 因此通过解释方差的比例来衡量未见过的样本可能被模型预测的程度。
 
from sklearn.metrics import r2_score
y_true = [3, -0.5, 2, 7]
y_pred = [2.5, 0.0, 2, 8]
print(r2_score(y_true, y_pred))
 
print("### 8. 回归预测的mean_absolute_error")
from sklearn.metrics import mean_absolute_error
y_true = [3, -0.5, 2, 7]
y_pred = [2.5, 0.0, 2, 8]
print(mean_absolute_error(y_true, y_pred))
 
print("### 9. 回归预测的mean_squared_error ")
from sklearn.metrics import mean_squared_error
y_true = [3, -0.5, 2, 7]
y_pred = [2.5, 0.0, 2, 8]
print(mean_squared_error(y_true, y_pred))
 
print("### 10. 回归预测的mean_squared_log_error")
from sklearn.metrics import mean_squared_log_error
y_true = [3, 5, 2.5, 7]
y_pred = [2.5, 5, 4, 8]
print(mean_squared_log_error(y_true, y_pred))
 
print("### 11.无监督聚类的Silhouette Coefficient")
from sklearn import metrics
from sklearn import datasets
import numpy as np
X, y = datasets.load_iris(return_X_y=True)
 
from sklearn.cluster import KMeans
kmeans_model = KMeans(n_clusters=3, random_state=1).fit(X)
labels = kmeans_model.labels_
print(metrics.silhouette_score(X, labels, metric='euclidean'))