Feature_selection

记录下最近在看的两个博客
刘建平Pinard
Machine Learning Mastery
参考了Feature Selection For Machine Learning in Python

数据文件仍然使用了上一篇中的pima-indians-diabetes.csv 才有了我这篇文章，在他的４种method之外，我多加了一个RandomForestClassifier,但是RandomForestClassifier和ExtraTreesClassifier结果非常接近.

代码如下：

点击(此处)折叠或打开

1. # coding: utf-8
   
2. import pandas as pd, numpy as np
   
3. import statsmodels.api as sm
   
4. import statsmodels.formula.api as smf
   
5. import seaborn as sns
   
6. import matplotlib.pyplot as plt
   
7. from sklearn.model_selection import train_test_split
   
8. from sklearn.svm import SVC
   
9. from sklearn.neighbors import KNeighborsClassifier
   
10. from sklearn.preprocessing import StandardScaler
    
11. from sklearn.metrics import (accuracy_score, f1_score, confusion_matrix)
    
12. from sklearn.ensemble import (RandomForestClassifier, ExtraTreesClassifier)
    
13. sns.set()
    
14. 
    
15. #Handle data
    
16. col_names=['preg','plas','pres','skin','insu','mass','pedi','age','class']
    
17. data=pd.read_csv('data/pima-indians-diabetes.csv',header=None,names=col_names)
    
18. X=data.drop('class', axis=1)
    
19. y=data['class']
    
20. #pring X.shape and y.shape
    
21. print(X.shape, y.shape,'\n')
    
22. 
    
23. #using PCA -- Principal Component Analysis
    
24. def pca_select():
    
25.     from sklearn.decomposition import PCA
    
26. 
    
27.     pca=PCA()
    
28.     pca.fit(X)
    
29.     #sum of this list almost == 1
    
30.     print(sum(pca.explained_variance_ratio_))
    
31. 
    
32.     #print the features according to their importance
    
33.     print("Print the feature importance from biggest to smallest")
    
34.     for i in pca.explained_variance_ratio_:
    
35.         print('{:.6f}'.format(i))
    
36.     
    
37.     #or use map
    
38.     '''
    
39.     for i in map('{:.6f}'.format , pca.explained_variance_ratio_):
    
40.         print(i,end='\t')
    
41.     '''
    
42. 
    
43.     print()
    
44. pca_select()
    
45. 
    
46. 
    
47. from sklearn.feature_selection import (f_classif,SelectKBest)
    
48. def univariate_select():
    
49.     
    
50.     test=SelectKBest(score_func=f_classif, k=4)
    
51.     fit=test.fit(X, y)
    
52.     
    
53.     print("feature scores: {}".format(fit.scores_))
    
54.     
    
55.     #a higher score means higher importance
    
56.     for score, feature in sorted(zip(fit.scores_, list(X))):
    
57.         print('score {:.format(score, feature))
    
58. 
    
59.     features=fit.transform(X)
    
60.     #print first 5 line of these 4 features
    
61.     print(features[0:5, :])
    
62.     
    
63.     #confirm the 4 features being selected are "preg", "plas", "mass", "age",
    
64.     assert np.array_equal(features, np.array(X[["preg", "plas", "mass", "age", ]]))
    
65.     
    
66.     
    
67. 
    
68. univariate_select()
    
69.     
    
70. from sklearn.feature_selection import RFE
    
71. from sklearn.linear_model import LogisticRegression
    
72. def RFE_select():
    
73.     model = LogisticRegression(solver='lbfgs')
    
74.     rfe=RFE(model, 4)
    
75.     fit=rfe.fit(X, y)
    
76.     
    
77.     print('\n')
    
78.     print("Num Features: {}".format(fit.n_features_))
    
79.     print("Selected Features: {}".format(fit.support_))
    
80.     print("Feature ranking, those with rank 1 will be choosen:\n{}".format(fit.ranking_))
    
81. 
    
82.     #print the 4 feature names to be choosen
    
83.     print("The 4 features to be choosen:")
    
84.     for idx,value in enumerate(fit.ranking_):
    
85.         if value==1:
    
86.             print(list(X)[idx], end='\t')
    
87.         
    
88.         
    
89. RFE_select()
    
90. 
    
91. 
    
92. def rlf_select():
    
93.     rnd_clf = RandomForestClassifier(random_state=0, n_estimators=100)
    
94.     rnd_clf.fit(X, y)
    
95.     rnd_name=rnd_clf.__class__.__name__
    
96.     
    
97.     feature_importances = rnd_clf.feature_importances_
    
98.     importance = sorted(zip(feature_importances, list(X)), reverse=True)
    
99.     
    
100.     print('\n\n{} most important features ( {} )'.format(4, rnd_name))
     
101.     print("The 4 features to be choosen:")
     
102.     [print(row) for i, row in enumerate(importance) if i < 4]
     
103. 
     
104. rlf_select()
     
105. 
     
106. def et_select():
     
107.     et_clf = ExtraTreesClassifier(random_state=0, n_estimators=100)
     
108.     et_clf.fit(X, y)
     
109.     et_name=et_clf.__class__.__name__
     
110.     
     
111.     feature_importances = et_clf.feature_importances_
     
112.     importance = sorted(zip(feature_importances, list(X)), reverse=True)
     
113.     
     
114.     print('\n\n{} most important features ( {} )'.format(4, et_name))
     
115.     print("The 4 features to be choosen:")
     
116.     [print(row) for i, row in enumerate(importance) if i < 4]
     
117. 
     
118. et_select()