logic error debugging

Author: theanti9

PyCrawler.py

@@ -12,20 +12,22 @@
 print"Continuing without psyco JIT compilation!"
 
 """
-The program should take 3 arguments
+The program should take arguments
 1) database file name
 2) start url
 3) crawl depth 
 Start out by checking to see if the args are there and
 set them to their variables
 """
-iflen(sys.argv) <4:
+iflen(sys.argv) <5:
 sys.exit("Not enough arguments!")
 else:
 dbname=sys.argv[1]
 starturl=sys.argv[2]
 crawldepth=int(sys.argv[3])
 
+# urlparse the start url
+surlparsed=urlparse.urlparse(starturl)
 
 # Connect to the db and create the tables if they don't already exist
 connection=sqlite.connect(dbname)
@@ -35,23 +37,6 @@
 cursor.execute('CREATE TABLE IF NOT EXISTS status ( s INTEGER, t TEXT )')
 connection.commit()
 
-"""
-# Check for a start point
-if len(sys.argv) < 2:
- print "No starting point! Checking existing queue"
- cursor.execute("SELECT * FROM queue LIMIT 1")
- c = cursor.fetchone()
- if c == None:
- sys.exit("ERROR: No start point! Exiting")
-else:
- try:
- if sys.argv[1]:
- cursor.execute("INSERT INTO queue VALUES ( (?) )", (sys.argv[1], ))
- connection.commit()
- except:
- pass
-"""
-
 # Compile keyword and link regex expressions
 keywordregex=re.compile('<meta\sname=["\']keywords["\']\scontent=["\'](.*?)["\']\s/>')
 linkregex=re.compile('<a\s*href=[\'|"](.*?)[\'"].*?>')
@@ -112,11 +97,12 @@ def crawl(self, crawling):
 # Load the link
 response=urllib2.urlopen(curl)
 except:
-# If it doesn't load, kill the function
+# If it doesn't load, skip this url
 return
 # Read response
 msg=response.read()
 
+# Find what's between the title tags
 startPos=msg.find('<title>')
 ifstartPos!=-1:
 endPos=msg.find('</title>', startPos+7)
@@ -142,13 +128,14 @@ def crawl(self, crawling):
 defqueue_links(self, url, links, cid, curdepth):
 ifcurdepth<crawldepth:
 # Read the links and inser them into the queue
-forlinkin(links.pop(0) for_inxrange(len(links))):
+forlinkinlinks:
 iflink.startswith('/'):
 link='http://'+url[1] +link
 eliflink.startswith('#'):
 link='http://'+url[1] +url[2] +link
 elifnotlink.startswith('http'):
 link='http://'+url[1] +'/'+link
+
 iflink.decode('utf-8') notincrawled:
 try:
 cursor.execute("INSERT INTO queue VALUES ( (?), (?), (?), (?) )", (None, cid, curdepth+1, link))