Python_Data_Science_第10课

#Bayesian Methods to create Anti-spammer
We can construct P(Spam | Word) for every (meaningful) word we encounter
during training.
Then multiply these together when analyzing a new mail to get the probability of it being spam.
Assumes the presence of different words are independent of each other - one reason this is called "Naive Bayes"

理论就是:  不考虑词和词之间的关系，简单的将每个词贡献的'spam‘值算出来，最后根据所有的这些词贡献出的'spam'值来分析新的邮件。

下面则是代码
首先是使用pandas读入数据，然后使用scikit-learn 来build 一个spam classifier, 最后使用这个spam classifier 来predict两个字符串到底应该归类spam 或者ham.

点击(此处)折叠或打开

1. #!/usr/bin/env python3
   
2. # -*- coding: utf-8 -*-
   
3. # Author: hezhb
   
4. # Created Time: Tue 01 May 2018 11:49:35 AM CST
   
5. 
   
6. import os
   
7. import io
   
8. import numpy as np
   
9. from pandas import DataFrame
   
10. from sklearn.feature_extraction.text import CountVectorizer
    
11. from sklearn.naive_bayes import MultinomialNB
    
12. 
    
13. def readFiles(path):
    
14.     for root, dirnames, filenames in os.walk(path):
    
15.         for filename in filenames:
    
16.             path = os.path.join(root, filename)
    
17.     
    
18.             inBody = False
    
19.             lines = []
    
20.             
    
21.             f = io.open(path, 'r', encoding='latin1')
    
22.             for line in f:
    
23.                 if inBody:
    
24.                     lines.append(line)
    
25.                 elif line == '\n':
    
26.                     inBody = True
    
27.                 
    
28.             f.close()
    
29.             message = '\n'.join(lines)
    
30.             yield path, message
    
31.     
    
32. 
    
33. def dataFrameFromDirectory(path, classification):
    
34.     rows = []
    
35.     index = []
    
36.                         
    
37.     for filename, message in readFiles(path):
    
38.         rows.append({'message':message, 'class':classification})
    
39.         index.append(filename)
    
40.     
    
41.     return DataFrame(rows, index=index)
    
42. 
    
43. 
    
44. PATH='./hands-on/emails/'
    
45. 
    
46. 
    
47. data = DataFrame({'message':[], 'class':[]})
    
48. data = data.append(dataFrameFromDirectory(PATH+'spam', 'spam'))
    
49. data = data.append(dataFrameFromDirectory(PATH+'ham', 'ham'))
    
50. #print(data.head())
    
51. 
    
52. """
    
53. Now we will use CountVectorizer to split up each message into its list of words
    
54. and throw that into a MultinomialNB classifier, call fit() and we've got
    
55. a trained spam filter ready to go.
    
56. """
    
57. 
    
58. vectorizer = CountVectorizer(encoding='latin1')
    
59. counts = vectorizer.fit_transform(data['message'].values)
    
60. classifier = MultinomialNB()
    
61. targets = data['class'].values
    
62. classifier.fit(counts, targets)
    
63. 
    
64. #Now can try this classifier out
    
65. examples = ['Free viagra Now', 'Hi Bob, how about a game of golf tommorrow.']
    
66. example_counts = vectorizer.transform(examples)
    
67. predictions = classifier.predict(example_counts)
    
68. print(predictions)