python - 加载腌制的分类器数据: Vocabulary not fitted Error

我在这里阅读了所有相关问题，但找不到可行的解决方案:

我的分类器创建:

class StemmedTfidfVectorizer(TfidfVectorizer):
    def build_analyzer(self):
        analyzer = super(TfidfVectorizer, self).build_analyzer()
        return lambda doc: english_stemmer.stemWords(analyzer(doc))

tf = StemmedTfidfVectorizer(analyzer='word', ngram_range=(1,2), min_df = 0, max_features=200000, stop_words = 'english')


def create_tfidf(f):
    docs = []
    targets = []
    with open(f, "r") as sentences_file:
        reader = csv.reader(sentences_file, delimiter=';')
        reader.next()
        for row in reader:
            docs.append(row[1])
            targets.append(row[0])

    tfidf_matrix = tf.fit_transform(docs)
    print tfidf_matrix.shape
    # print tf.get_feature_names()
    return tfidf_matrix, targets


X,y = create_tfidf("l0.csv")
clf = LinearSVC().fit(X,y)

_ = joblib.dump(clf, 'linearL0_3gram_100K.pkl', compress=9)

该位有效，并生成.pkl，然后我尝试在其他脚本中使用它:

class StemmedTfidfVectorizer(TfidfVectorizer):
    def build_analyzer(self):
        analyzer = super(TfidfVectorizer, self).build_analyzer()
        return lambda doc: english_stemmer.stemWords(analyzer(doc))

tf = StemmedTfidfVectorizer(analyzer='word', ngram_range=(1,2), min_df = 0, max_features=200000, stop_words = 'english')


clf = joblib.load('linearL0_3gram_100K.pkl')

print clf
test = "My super elaborate test string to test predictions"
print test + clf.predict(tf.transform([test]))[0]

我得到ValueError:Vocabulary wasn't fitted or is empty!
编辑:根据请求的错误回溯

 File "classifier.py", line 27, in <module>
    print test + clf.predict(tf.transform([test]))[0]
  File "/home/ec2-user/.local/lib/python2.7/site-packages/sklearn/feature_extraction/text.py", line 1313, in transform
    X = super(TfidfVectorizer, self).transform(raw_documents)
  File "/home/ec2-user/.local/lib/python2.7/site-packages/sklearn/feature_extraction/text.py", line 850, in transform
    self._check_vocabulary()
  File "/home/ec2-user/.local/lib/python2.7/site-packages/sklearn/feature_extraction/text.py", line 271, in _check_vocabulary
    check_is_fitted(self, 'vocabulary_', msg=msg),
  File "/home/ec2-user/.local/lib/python2.7/site-packages/sklearn/utils/validation.py", line 627, in check_is_fitted
    raise NotFittedError(msg % {'name': type(estimator).__name__})
sklearn.utils.validation.NotFittedError: StemmedTfidfVectorizer - Vocabulary wasn't fitted.

最佳答案

好的，我通过使用管道将矢量化程序保存在.plk中来解决了该问题

这是它的外观(也更简单):

from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.externals import joblib
from sklearn.pipeline import Pipeline
import Stemmer
import pickle

english_stemmer = Stemmer.Stemmer('en')


class StemmedTfidfVectorizer(TfidfVectorizer):
    def build_analyzer(self):
        analyzer = super(TfidfVectorizer, self).build_analyzer()
        return lambda doc: english_stemmer.stemWords(analyzer(doc))


def create_tfidf(f):
    docs = []
    targets = []
    with open(f, "r") as sentences_file:
        reader = csv.reader(sentences_file, delimiter=';')
        reader.next()
        for row in reader:
            docs.append(row[1])
            targets.append(row[0])
    return docs, targets


docs,y = create_tfidf("l1.csv")
tf = StemmedTfidfVectorizer(analyzer='word', ngram_range=(1,2), min_df = 0, max_features=200000, stop_words = 'english')
clf = LinearSVC()

vec_clf = Pipeline([('tfvec', tf), ('svm', clf)])

vec_clf.fit(docs,y)

_ = joblib.dump(vec_clf, 'linearL0_3gram_100K.pkl', compress=9)

在另一边:

from sklearn.svm import LinearSVC
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.externals import joblib
import Stemmer
import pickle

english_stemmer = Stemmer.Stemmer('en')

class StemmedTfidfVectorizer(TfidfVectorizer):
    def build_analyzer(self):
        analyzer = super(TfidfVectorizer, self).build_analyzer()
        return lambda doc: english_stemmer.stemWords(analyzer(doc))


clf = joblib.load('linearL0_3gram_100K.pkl')
test = ["My super elaborate test string to test predictions"]
print test + clf.predict(test)[0]

重要事项:

转换器是tf的一部分，因此是管道的一部分，因此无需重新声明新的矢量化程序(这是较早的失败点，因为它需要训练后的数据中的词汇表)，也无需对测试字符串进行.transform()。

关于python - 加载腌制的分类器数据: Vocabulary not fitted Error，我们在Stack Overflow上找到一个类似的问题：https://stackoverflow.com/questions/31744519/