我正在尝试预测某些文档的标签。每个文档可以具有多个标签。这是我写的一个示例程序
import pandas as pd
import pickle
import re
from sklearn.cross_validation import train_test_split
from sklearn.metrics.metrics import classification_report, accuracy_score, confusion_matrix
from nltk.stem import WordNetLemmatizer
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.naive_bayes import MultinomialNB as MNB
from sklearn.pipeline import Pipeline
from sklearn.grid_search import GridSearchCV
def Mytrain():
pipeline = Pipeline([
('vect', TfidfVectorizer(stop_words='english',sublinear_tf=True)),
('clf', MNB())
])
parameters = {
'vect__max_df': (0.25, 0.5, 0.6, 0.7, 1.0),
'vect__ngram_range': ((1, 1), (1, 2), (2,3), (1,3), (1,4), (1,5)),
'vect__use_idf': (True, False),
'clf__fit_prior': (True, False)
}
traindf = pickle.load(open("train.pkl","rb"))
X, y = traindf['Data'], traindf['Tags'].as_matrix()
Xtrain, Xtest, ytrain, ytest = train_test_split(X, y, train_size=0.7)
gridSearch = GridSearchCV(pipeline, parameters, n_jobs=3, verbose=1, scoring='accuracy')
gridSearch.fit(Xtrain, ytrain)
print ('best score: %0.3f' % gridSearch.best_score_)
print ('best parameters set:')
res = open("res.txt", 'w')
res.write ('best parameters set:\n')
bestParameters = gridSearch.best_estimator_.get_params()
for paramName in sorted(parameters.keys()):
print ('\t %s: %r' % (paramName, bestParameters[paramName]))
res.write('\t %s: %r\n' % (paramName, bestParameters[paramName]))
pickle.dump(bestParameters,open("bestParams.pkl","wb"))
predictions = gridSearch.predict(Xtest)
print ('Accuracy:', accuracy_score(ytest, predictions))
print ('Confusion Matrix:', confusion_matrix(ytest, predictions))
print ('Classification Report:', classification_report(ytest, predictions))
请注意，标签可以有多个值。现在我明白了
An unexpected error occurred while tokenizing input
The following traceback may be corrupted or invalid
The error message is: ('EOF in multi-line statement', (40, 0))
Traceback (most recent call last):
File "X:\abc\predMNB.py", line 128, in <module>
MNBdrill(fname,topn)
File "X:\abc\predMNB.py", line 82, in MNBdrill
gridSearch.fit(Xtrain, ytrain)
File "X:\pqr\Anaconda2\lib\site-packages\sklearn\grid_search.py", line 732, in fit
return self._fit(X, y, ParameterGrid(self.param_grid))
File "X:\pqr\Anaconda2\lib\site-packages\sklearn\grid_search.py", line 505, in _fit
for parameters in parameter_iterable
File "X:\pqr\Anaconda2\lib\site-packages\sklearn\externals\joblib\parallel.py", line 666, in __call__
self.retrieve()
File "X:\pqr\Anaconda2\lib\site-packages\sklearn\externals\joblib\parallel.py", line 549, in retrieve
raise exception_type(report)
sklearn.externals.joblib.my_exceptions.JoblibMemoryError: JoblibMemoryError
然后
Multiprocessing exception:
...........................................................................
X:\pqr\Anaconda2\lib\site-packages\sklearn\grid_search.py in fit(self=GridSearchCV(cv=None, error_score='raise',
..._func=None,
scoring='accuracy', verbose=1), X=14151 text for document having t1,t2,t3,t4
Name: Content, dtype: object, y=array([u't1',u't2',u't3',u't4'], dtype=object))
727 y : array-like, shape = [n_samples] or [n_samples, n_output], optional
728 Target relative to X for classification or regression;
729 None for unsupervised learning.
730
731 """
--> 732 return self._fit(X, y, ParameterGrid(self.param_grid))
self._fit = <bound method GridSearchCV._fit of GridSearchCV(...func=None,
scoring='accuracy', verbose=1)>
X = 14151 text for document having t1,t2,t3,t4
Name: Content, dtype: object
y = array([u't1',u't2',u't3',u't4'], dtype=object)
self.param_grid = {'clf__fit_prior': (True, False), 'vect__max_df': (0.25, 0.5, 0.6, 0.7, 1.0), 'vect__ngram_range': ((1, 1), (1, 2), (2, 3), (1, 3), (1, 4), (1, 5)), 'vect__use_idf': (True, False)}
733
734
735 class RandomizedSearchCV(BaseSearchCV):
736 """Randomized search on hyper parameters.
...........................................................................
X:\pqr\Anaconda2\lib\site-packages\sklearn\grid_search.py in _fit(self=GridSearchCV(cv=None, error_score='raise',
..._func=None,
scoring='accuracy', verbose=1), X=14151 text for document having t1,t2,t3,t4
Name: Content, dtype: object, y=array([u't1',u't2',u't3',u't4'], dtype=object), parameter_iterable=<sklearn.grid_search.ParameterGrid object>)
500 )(
501 delayed(_fit_and_score)(clone(base_estimator), X, y, self.scorer_,
502 train, test, self.verbose, parameters,
503 self.fit_params, return_parameters=True,
504 error_score=self.error_score)
--> 505 for parameters in parameter_iterable
parameters = undefined
parameter_iterable = <sklearn.grid_search.ParameterGrid object>
506 for train, test in cv)
507
508 # Out is a list of triplet: score, estimator, n_test_samples
509 n_fits = len(out)
...........................................................................
X:\pqr\Anaconda2\lib\site-packages\sklearn\externals\joblib\parallel.py in __call__(self=Parallel(n_jobs=3), iterable=<itertools.islice object>)
661 if pre_dispatch == "all" or n_jobs == 1:
662 # The iterable was consumed all at once by the above for loop.
663 # No need to wait for async callbacks to trigger to
664 # consumption.
665 self._iterating = False
--> 666 self.retrieve()
self.retrieve = <bound method Parallel.retrieve of Parallel(n_jobs=3)>
667 # Make sure that we get a last message telling us we are done
668 elapsed_time = time.time() - self._start_time
669 self._print('Done %3i out of %3i | elapsed: %s finished',
670 (len(self._output),
---------------------------------------------------------------------------
Sub-process traceback:
---------------------------------------------------------------------------
MemoryError
之后，堆栈跟踪将继续指向具有相同问题的其他方法。如果需要，我可以发布整件事，但这是我认为正在发生的事情
注意
scoring='accuracy', verbose=1), X=14151 text for document having t1,t2,t3,t4
Name: Content, dtype: object, y=array([u't1',u't2',u't3',u't4'], dtype=object))
因为有多个标签，这可能会引起问题吗？
还有，这是什么意思
多处理异常？
MemoryError？
这个你能帮我吗。
最佳答案
您有多少火车数据？
我最好的选择是，唯一的“真实”错误是MemoryError，即您在尝试训练分类器时使用了所有可用的RAM，而其他所有奇怪的错误/回溯都是内存分配失败的结果。
训练分类器时，您是否检查过您的可用内存？