edit link extractor tutorial

x4nth055 · x4nth055 · commit 406ef1d1202a · 2021-04-11T23:41:47.000+01:00
diff --git a/web-scraping/link-extractor/link_extractor.py b/web-scraping/link-extractor/link_extractor.py
@@ -9,6 +9,7 @@
 GREEN = colorama.Fore.GREEN
 GRAY = colorama.Fore.LIGHTBLACK_EX
 RESET = colorama.Fore.RESET
+YELLOW = colorama.Fore.YELLOW
 
 # initialize the set of links (unique links)
 internal_urls = set()
@@ -62,7 +63,7 @@ def get_all_website_links(url):
 return urls
 
 
-def crawl(url, max_urls=50):
+def crawl(url, max_urls=30):
 """
 Crawls a web page and extracts all links.
 You'll find all links in `external_urls` and `internal_urls` global set variables.
@@ -71,6 +72,7 @@ def crawl(url, max_urls=50):
 """
 global total_urls_visited
 total_urls_visited += 1
+ print(f"{YELLOW}[*] Crawling:{url}{RESET}")
 links = get_all_website_links(url)
 for link in links:
 if total_urls_visited > max_urls:
@@ -93,6 +95,7 @@ def crawl(url, max_urls=50):
 print("[+] Total Internal links:", len(internal_urls))
 print("[+] Total External links:", len(external_urls))
 print("[+] Total URLs:", len(external_urls) + len(internal_urls))
+ print("[+] Total crawled URLs:", max_urls)
 
 domain_name = urlparse(url).netloc
 
diff --git a/web-scraping/link-extractor/link_extractor_js.py b/web-scraping/link-extractor/link_extractor_js.py
@@ -9,6 +9,7 @@
 GREEN = colorama.Fore.GREEN
 GRAY = colorama.Fore.LIGHTBLACK_EX
 RESET = colorama.Fore.RESET
+YELLOW = colorama.Fore.YELLOW
 
 # initialize the set of links (unique links)
 internal_urls = set()
@@ -71,7 +72,7 @@ def get_all_website_links(url):
 return urls
 
 
-def crawl(url, max_urls=50):
+def crawl(url, max_urls=30):
 """
 Crawls a web page and extracts all links.
 You'll find all links in `external_urls` and `internal_urls` global set variables.
@@ -80,6 +81,7 @@ def crawl(url, max_urls=50):
 """
 global total_urls_visited
 total_urls_visited += 1
+ print(f"{YELLOW}[*] Crawling:{url}{RESET}")
 links = get_all_website_links(url)
 for link in links:
 if total_urls_visited > max_urls:
@@ -102,6 +104,7 @@ def crawl(url, max_urls=50):
 print("[+] Total Internal links:", len(internal_urls))
 print("[+] Total External links:", len(external_urls))
 print("[+] Total URLs:", len(external_urls) + len(internal_urls))
+ print("[+] Total crawled URLs:", max_urls)
 
 domain_name = urlparse(url).netloc
 

-Original file line number
+Diff line change
 GREEN=colorama.Fore.GREEN
 GRAY=colorama.Fore.LIGHTBLACK_EX
 RESET=colorama.Fore.RESET
 +YELLOW=colorama.Fore.YELLOW
 # initialize the set of links (unique links)
 internal_urls=set()
 returnurls
 -defcrawl(url, max_urls=50):
 +defcrawl(url, max_urls=30):
 """
  Crawls a web page and extracts all links.
  You'll find all links in `external_urls` and `internal_urls` global set variables.
  """
 globaltotal_urls_visited
 total_urls_visited+=1
 +print(f"{YELLOW}[*] Crawling: {url}{RESET}")
 links=get_all_website_links(url)
 forlinkinlinks:
 iftotal_urls_visited>max_urls:
 print("[+] Total Internal links:", len(internal_urls))
 print("[+] Total External links:", len(external_urls))
 print("[+] Total URLs:", len(external_urls) +len(internal_urls))
 +print("[+] Total crawled URLs:", max_urls)
 domain_name=urlparse(url).netloc