Modify UnicodeDecodeError text. You'll use utf-8

neoneo40 · neoneo40 · commit d8181339aaa4 · 2015-01-27T10:05:21.000+09:00
diff --git a/ch05/classify.py b/ch05/classify.py
@@ -54,6 +54,7 @@ def prepare_sent_features():
 if not text:
 meta[pid]['AvgSentLen'] = meta[pid]['AvgWordLen'] = 0
 else:
+ text = text.decode('utf-8')
 sent_lens = [len(nltk.word_tokenize(
 sent)) for sent in nltk.sent_tokenize(text)]
 meta[pid]['AvgSentLen'] = np.mean(sent_lens)

-Original file line number
+Diff line change
 ifnottext:
 meta[pid]['AvgSentLen'] =meta[pid]['AvgWordLen'] =0
 else:
 +text=text.decode('utf-8')
 sent_lens= [len(nltk.word_tokenize(
 sent)) forsentinnltk.sent_tokenize(text)]
 meta[pid]['AvgSentLen'] =np.mean(sent_lens)