4、Iris数据集上的非嵌套和嵌套交叉验证
from sklearn.datasets import load_iris
from matplotlib import pyplot as plt
from sklearn.svm import SVC
from sklearn.model_selection import GridSearchCV, cross_val_score, KFold
import numpy as np
plt.rcParams['font.sans-serif'] = ['SimHei']
plt.rcParams['axes.unicode_minus'] = False
# 随机试验次数
NUM_TRIALS = 30
# 导入数据集
iris = load_iris()
X_iris = iris.data
y_iris = iris.target
# 设置参数的可能值以优化
p_grid = {"C": [1, 10, 100],
"gamma": [.01, .1]}
# 我们将使用带有“ rbf”内核的支持向量分类器
svm = SVC(kernel="rbf")
# 存储分数的数组
non_nested_scores = np.zeros(NUM_TRIALS)
nested_scores = np.zeros(NUM_TRIALS)
# 每次试用循环
for i in range(NUM_TRIALS):
# 独立于数据集，为内部和外部循环选择交叉验证技术。
# 例如“ GroupKFold”，“ LeaveOneOut”，“ LeaveOneGroupOut”等。
inner_cv = KFold(n_splits=4, shuffle=True, random_state=i)
outer_cv = KFold(n_splits=4, shuffle=True, random_state=i)
# 非嵌套参数搜索和评分
clf = GridSearchCV(estimator=svm, param_grid=p_grid, cv=inner_cv)
clf.fit(X_iris, y_iris)
non_nested_scores[i] = clf.best_score_
# 带有参数优化的嵌套简历
nested_score = cross_val_score(clf, X=X_iris, y=y_iris, cv=outer_cv)
nested_scores[i] = nested_score.mean()
score_difference = non_nested_scores - nested_scores
print("Average difference of {:6f} with std. dev. of {:6f}."
.format(score_difference.mean(), score_difference.std()))
# 嵌套和非嵌套CV在每个试验中的得分
plt.figure()
plt.subplot(211)
non_nested_scores_line, = plt.plot(non_nested_scores, color='r')
nested_line, = plt.plot(nested_scores, color='b')
plt.ylabel("score", fontsize="14")
plt.legend([non_nested_scores_line, nested_line],
["Non-Nested CV", "Nested CV"],
bbox_to_anchor=(0, .4, .5, 0))
plt.title("Iris数据集上的非嵌套和嵌套交叉验证",
x=.5, y=1.1, fontsize="15")
# 绘制差异柱状图
plt.subplot(212)
difference_plot = plt.bar(range(NUM_TRIALS), score_difference)
plt.xlabel("Individual Trial #")
plt.legend([difference_plot],
["Non-Nested CV - Nested CV Score"],
bbox_to_anchor=(0, 1, .8, 0))
plt.ylabel("score difference", fontsize="14")
plt.show()
