Merge pull request theanti9#12 from EthanBlackburn/master

fixed content processor

Author: theanti9

content_processor.py

@@ -1,5 +1,5 @@
 frommultiprocessingimportPool
-importre, sys, logging
+importre, sys, logging, string
 
 fromready_queueimportready_queue
 
@@ -9,13 +9,21 @@ def rankKeywords(text):
 invalid_keywords= ['', ' ', "i", "a", "an", "and", "the", "for", "be", "to", "or", "too", "also"]
 ranks={}
 text=text.split(' ')
+exclude=set(string.punctuation)
 fortintext:
+#remove punctuation if attached to word
+temp=t
+t=''
+foriinrange(len(temp)):
+if(temp[i] notinexclude):
+t+=temp[i]
+t=t.strip()
 iftininvalid_keywords:
 continue
 ifnotranks.has_key(t):
 ranks[t] =1
 else:
-ranks[t] +=1
+ranks[t] +=1
 returnranks
 
 defstripPunctuation(text):
@@ -83,13 +91,18 @@ def processBody(self):
 offset=0
 i=0
 l= []
-whileTrue:
+cont=True
+whilecont:
+#this divides the text into sets of 500 words
+#set j to the index of the last letter of the 500th word
 j=self.findnth(self.text[i:],' ',500)
-offset+=j
+#if only 500 words or less are left
 ifj==-1:
-break
-l.append(self.text[i:j])
-i=offset+j+1
+cont=False
+#Should append a string that contains 500 words for each loop(except the last loop) to l
+#last loop should append a string with 500 words or less to l
+l.append(self.text[i:i+j])
+i+=j+1
 logger.debug("processing with %i threads"%len(l))
 try:
 iflen(l) ==0:
@@ -136,4 +149,4 @@ def getDataDict(self):
 fork,vinself.keywords.items():
 ifv<3:
 delself.keywords[k]
-return{"address":self.url, "title":self.title, "status":self.status, "size":self.size, "keywords":self.keywords}
+return{"address":self.url, "title":self.title, "status":self.status, "size":self.size, "keywords":self.keywords}