Work on colors, Added option to rotate sqlite database files

Author: WilHall

PyCrawler.db.1

@@ -1,7 +1,7 @@
 fromqueryimportCrawlerDb
 fromcontent_processorimportContentProcessor
-fromsettingsimportVERBOSE, COLOR_ERROR, COLOR_SUCCESS
-importsys, urlparse, urllib2
+fromsettingsimportVERBOSE, USE_COLORS, DATABASE_ENGINE, DATABASE_NAME, SQLITE_ROTATE_DATABASE_ON_STARTUP
+importsys, urlparse, urllib2, shutil, glob, robotparser
 importcPrinter
 
 # ===== Init stuff =====
@@ -14,36 +14,58 @@
 processor=ContentProcessor(None, None, None)
 
 # get cprinter
-printer=cPrinter.Printer(COLOR_SUCCESS, COLOR_ERROR)
+printer=cPrinter.Printer(USE_COLORS)
+
+# robot parser init
+robot=robotparser.RobotFileParser()
 
 iflen(sys.argv) <2:
-printer.p("Error: No start url was passed", printer.error)
+printer.p("Error: No start url was passed", printer.other)
 sys.exit()
 
 l=sys.argv[1:]
 
 cdb.enqueue(l)
 
 defcrawl():
-printer.p("starting...", printer.success)
-queue_empty=False
+printer.p("starting...", printer.other)
 whileTrue:
 url=cdb.dequeue()
+u=urlparse.urlparse(url)
+robot.set_url('http://'+u[1]+"/robots.txt")
+ifnotrobot.can_fetch('PyCrawler', url):
+printer.p("Url disallowed by robots.txt: %s "%url, printer.other)
+continue
+ifnoturl.startswith('http'):
+printer.p("Unfollowable link found at %s "%url, printer.other)
+continue
+
 ifcdb.checkCrawled(url):
 continue
 ifurlisFalse:
-queue_empty=True
+break
 status=0
 request=None
 try:
 request=urllib2.urlopen(str(url))
 excepturllib2.URLError, e:
 printer.p(e.reason, printer.error)
+printer.p("Exception at url: %s"%url, printer.error)
+
+continue
 excepturllib2.HTTPError, e:
 status=e.code
 ifstatus==0:
 status=200
 data=request.read()
+processor.setInfo(str(url), status, data)
+ret=processor.process()
+ifstatus!=200:
+continue
+add_queue= []
+forqinret:
+ifnotcdb.checkCrawled(q):
+add_queue.append(q)
 
 processor.setInfo(str(url), status, data)
 add_queue=processor.process()
@@ -52,17 +74,26 @@ def crawl():
 printer.p("Got %s status from %s"% (status, url), printer.success)
 printer.p("Found %i links"%l, printer.success)
 ifl>0:
-ifqueue_empty==True:
-queue_empty=False
 cdb.enqueue(add_queue) 
 cdb.addPage(processor.getDataDict())
 processor.reset()
-ifqueue_empty:
-break
 
-printer.p("finishing...", printer.success)
+printer.p("finishing...", printer.other)
 cdb.close()
 printer.p("done! goodbye!", printer.success)
 
 if__name__=="__main__":
-crawl()
+ifDATABASE_ENGINE=="sqlite"andSQLITE_ROTATE_DATABASE_ON_STARTUP:
+dbs=glob.glob("*.db*")
+index=1;
+while("%s.db.%s"% (DATABASE_NAME, index) indbs):
+index+=1
+shutil.copy2(dbs[len(dbs)-1], "%s.db.%s"% (DATABASE_NAME, index))
+try:
+crawl()
+exceptKeyboardInterrupt:
+printer.p("Stopping", printer.error)
+sys.exit()
+exceptException, e:
+printer.p("EXCEPTION: %s "%e, printer.error)
+

PyCrawler.py

@@ -2,10 +2,13 @@
 
 classPrinter():
 
-def__init__(self, COLOR_SUCCESS, COLOR_ERROR):
+def__init__(self, USE_COLORS):
 # Define our types
 self.success=0;
 self.error=1;
+self.other=2;
+
+self.USE_COLORS=USE_COLORS
 
 # Initialize environment
 curses.setupterm()
@@ -16,19 +19,18 @@ def __init__(self, COLOR_SUCCESS, COLOR_ERROR):
 #Get the normal attribute
 self.COLOR_NORMAL=curses.tigetstr('sgr0')
 
-# Initialize custom colors to the first two slots
-curses.initscr()
-curses.start_color()
-curses.init_color(0, COLOR_SUCCESS[0], COLOR_SUCCESS[1], COLOR_SUCCESS[2])
-curses.init_color(1, COLOR_ERROR[0], COLOR_ERROR[1], COLOR_ERROR[2])
-curses.endwin()
-
 # Get + Save the color sequences
-self.COLOR_SUCCESS=curses.tparm(self.fcap, 0)
-self.COLOR_ERROR=curses.tparm(self.fcap, 1)
+self.COLOR_SUCCESS=curses.tparm(self.fcap, curses.COLOR_GREEN)
+self.COLOR_ERROR=curses.tparm(self.fcap, curses.COLOR_RED)
+self.COLOR_OTHER=curses.tparm(self.fcap, curses.COLOR_YELLOW)
 
 defp(self, text, type):
-iftype==self.success:
-print"%s%s%s"% (self.COLOR_SUCCESS, text, self.COLOR_NORMAL)
-eliftype==self.error:
-print"%s%s%s"% (self.COLOR_SUCCESS, text, self.COLOR_NORMAL)
+ifself.USE_COLORS:
+iftype==self.success:
+print"%s[*] %s%s"% (self.COLOR_SUCCESS, text, self.COLOR_NORMAL)
+eliftype==self.error:
+print"%s[!] %s%s"% (self.COLOR_ERROR, text, self.COLOR_NORMAL)
+eliftype==self.other:
+print"%s[.] %s%s"% (self.COLOR_OTHER, text, self.COLOR_NORMAL)
+else:
+printtext

cPrinter.py

@@ -20,6 +20,7 @@ def rankKeywords(text):
 defstripPunctuation(text):
 pattern=re.compile(r'[^\w\s]')
 returnpattern.sub('', text)
+
 classContentProcessor:
 
 def__init__(self, url, status, text):
@@ -114,4 +115,7 @@ def process(self):
 returnqueue
 
 defgetDataDict(self):
+fork,vinself.keywords.items():
+ifv<3:
+delself.keywords[k]
 return{"address":self.url, "title":self.title, "status":self.status, "size":self.size, "keywords":self.keywords}

content_processor.py

@@ -56,7 +56,7 @@ def enqueue(self, urls):
 returnFalse
 iflen(urls) ==0:
 returnTrue
-args= [{'address':u} foruinurls]
+args= [{'address':unicode(u)} foruinurls]
 result=self.connection.execute(self.queue_table.insert(), args)
 ifresult:
 returnTrue
@@ -81,7 +81,7 @@ def dequeue(self):
 returnFalse
 
 defcheckCrawled(self, url):
-s=select([self.crawl_table]).where(self.crawl_table.c.address==url)
+s=select([self.crawl_table]).where(self.crawl_table.c.address==unicode(url))
 result=self.connection.execute(s)
 iflen(result.fetchall()) >0:
 result.close()
@@ -100,7 +100,7 @@ def addPage(self, data):
 ifnotself.connected:
 returnFalse
 # Add the page to the crawl table
-result=self.connection.execute(self.crawl_table.insert().values(address=data['address'],http_status=data['status'],title=data['title'],size=data['size']))
+result=self.connection.execute(self.crawl_table.insert().values(address=unicode(data['address']),http_status=data['status'],title=unicode(data['title']),size=data['size']))
 ifnotresult:
 returnFalse
 # generate list of argument dictionaries for the insert many statement

query.py

@@ -7,10 +7,8 @@
 DATABASE_USER=""# Not used with sqlite
 DATABASE_PASS=""# Not used with sqlite
 
-VERBOSE=True
+SQLITE_ROTATE_DATABASE_ON_STARTUP=True# Rotate the database to a new one on startup
 
-# These values are for the text output colors.
-# List values are 0-255 RGB values, respectively.
+VERBOSE=True
 
-COLOR_SUCCESS= [0, 255, 0] # Success Color (Green)
-COLOR_ERROR= [255, 0, 0] # Error Color (Red)
+USE_COLORS=True# Whether or not colors should be used when printing text