Revert "fixed content_processor"

This reverts commit 8ea6f69.

Author: Ethan Blackburn

PyCrawler.py

@@ -92,4 +92,4 @@ def crawl():
 exceptException, e:
 logger.error("EXCEPTION: %s "%e)
 traceback.print_exc()
-
+

content_processor.py

@@ -1,5 +1,5 @@
 frommultiprocessingimportPool
-importre, sys, logging, string
+importre, sys, logging
 
 fromready_queueimportready_queue
 
@@ -9,21 +9,13 @@ def rankKeywords(text):
 invalid_keywords= ['', ' ', "i", "a", "an", "and", "the", "for", "be", "to", "or", "too", "also"]
 ranks={}
 text=text.split(' ')
-exclude=set(string.punctuation)
 fortintext:
-#remove punctuation if attached to word
-temp=t
-t=''
-foriinrange(len(temp)):
-if(temp[i] notinexclude):
-t+=temp[i]
-t=t.strip()
 iftininvalid_keywords:
 continue
 ifnotranks.has_key(t):
 ranks[t] =1
 else:
-ranks[t] +=1
+ranks[t] +=1
 returnranks
 
 defstripPunctuation(text):
@@ -91,18 +83,13 @@ def processBody(self):
 offset=0
 i=0
 l= []
-cont=True
-whilecont:
-#this divides the text into sets of 500 words
-#set j to the index of the last letter of the 500th word
+whileTrue:
 j=self.findnth(self.text[i:],' ',500)
-#if only 500 words or less are left
+offset+=j
 ifj==-1:
-cont=False
-#Should append a string that contains 500 words for each loop(except the last loop) to l
-#last loop should append a string with 500 words or less to l
-l.append(self.text[i:i+j])
-i+=j+1
+break
+l.append(self.text[i:j])
+i=offset+j+1
 logger.debug("processing with %i threads"%len(l))
 try:
 iflen(l) ==0: