Modify UnicodeDecodeError text in Python 2.x

neoneo40 · neoneo40 · commit 4de7593567e9 · 2015-01-28T06:21:37.000+09:00
diff --git a/ch05/classify.py b/ch05/classify.py
@@ -54,7 +54,9 @@ def prepare_sent_features():
 if not text:
 meta[pid]['AvgSentLen'] = meta[pid]['AvgWordLen'] = 0
 else:
- text = text.decode('utf-8')
+ from platform import python_version
+ if python_version().startswith('2'):
+ text = text.decode('utf-8')
 sent_lens = [len(nltk.word_tokenize(
 sent)) for sent in nltk.sent_tokenize(text)]
 meta[pid]['AvgSentLen'] = np.mean(sent_lens)

-Original file line number
+Diff line change
 ifnottext:
 meta[pid]['AvgSentLen'] =meta[pid]['AvgWordLen'] =0
 else:
 -text=text.decode('utf-8')
 +fromplatformimportpython_version
 +ifpython_version().startswith('2'):
 +text=text.decode('utf-8')
 sent_lens= [len(nltk.word_tokenize(
 sent)) forsentinnltk.sent_tokenize(text)]
 meta[pid]['AvgSentLen'] =np.mean(sent_lens)