Merge pull request luispedro#6 from re4lfl0w/unicodedecodeerror_fix

wrichert · wrichert · commit e1d2b6d635e0 · 2015-02-03T16:19:18.000+08:00
Modify UnicodeDecodeError text. Use utf-8 in case of Python version is 2.x.
diff --git a/ch05/classify.py b/ch05/classify.py
@@ -54,6 +54,9 @@ def prepare_sent_features():
 if not text:
 meta[pid]['AvgSentLen'] = meta[pid]['AvgWordLen'] = 0
 else:
+ from platform import python_version
+ if python_version().startswith('2'):
+ text = text.decode('utf-8')
 sent_lens = [len(nltk.word_tokenize(
 sent)) for sent in nltk.sent_tokenize(text)]
 meta[pid]['AvgSentLen'] = np.mean(sent_lens)

-Original file line number
+Diff line change
 ifnottext:
 meta[pid]['AvgSentLen'] =meta[pid]['AvgWordLen'] =0
 else:
 +fromplatformimportpython_version
 +ifpython_version().startswith('2'):
 +text=text.decode('utf-8')
 sent_lens= [len(nltk.word_tokenize(
 sent)) forsentinnltk.sent_tokenize(text)]
 meta[pid]['AvgSentLen'] =np.mean(sent_lens)