added stock scraper, converted all scripts to python 2/3 compatibility

mjhea0 · mjhea0 · commit 0bb4c8c25500 · 2015-05-17T03:49:35.000-06:00
diff --git a/.gitignore b/.gitignore
@@ -1,3 +1,5 @@
 .pyc
 .DS_Store
-_tmp
+_tmp
+env
+__pycache__
diff --git a/02_find_all_links.py b/02_find_all_links.py
@@ -1,18 +1,18 @@
-import urllib2
+import requests
 import re
 
 # get url
-url =raw_input('Enter a URL (include `http://`): ')
+url = input('Enter a URL (include `http://`): ')
 
 # connect to the url
-website = urllib2.urlopen(url)
+website = requests.get(url)
 
 # read html
-html = website.read()
+html = website.text
 
 # use re.findall to grab all the links
 links = re.findall('"((http|ftp)s?://.*?)"', html)
 
 # output links
 for link in links:
-	print link[0]
+ print(link[0])
diff --git a/03_simple_twitter_manager.py b/03_simple_twitter_manager.py
@@ -1,25 +1,28 @@
 import twitter
- 
- 
+
+
 TWITTER_CONSUMER_KEY = 'XXX'
 TWITTER_CONSUMER_SECRET = 'XXX'
 TWITTER_ACCESS_TOKEN_KEY = 'XXX'
 TWITTER_ACCESS_TOKEN_SECRET = 'XXX'
- 
+
 twitter_api = twitter.Api(
 consumer_key=TWITTER_CONSUMER_KEY,
 consumer_secret=TWITTER_CONSUMER_SECRET,
 access_token_key=TWITTER_ACCESS_TOKEN_KEY,
 access_token_secret=TWITTER_ACCESS_TOKEN_SECRET
 )
- 
+
 if __name__ == '__main__':
 follower_ids = twitter_api.GetFollowerIDs()
 following_ids = twitter_api.GetFriendIDs()
- zombie_follows = [following_id for following_id in following_ids if following_id not in follower_ids]
- 
- confirm = raw_input("Are you sure you want to unfollow %s tweeps [y|n]? " % (len(zombie_follows)))
+ zombie_follows = [following_id for following_id in
+ following_ids if following_id not in follower_ids]
+
+ confirm = raw_input(
+ "Are you sure you want to unfollow{0} tweeps [y|n]? ".format(
+ (len(zombie_follows))))
 if confirm.lower() == 'y':
 for id in zombie_follows:
 user = twitter_api.DestroyFriendship(user_id=id)
- print "Unfollowed %s" % (user.screen_name)
+ print("Unfollowed{0}".format(user.screen_name))
diff --git a/04_rename_with_slice.py b/04_rename_with_slice.py
@@ -8,7 +8,7 @@
 new_file_name = file_name[:-6] + extension
 try:
 os.rename(file, new_file_name)
- except OSError, e:
- print e
+ except OSError as e:
+ print(e)
 else:
- print "Renamed{} to{}".format(file, new_file_name)
+ print("Renamed{} to{}".format(file, new_file_name))
diff --git a/05_load_json_without_dupes.py b/05_load_json_without_dupes.py
@@ -1,11 +1,9 @@
-import json
-
 def dict_raise_on_duplicates(ordered_pairs):
 """reject duplicate keys"""
 my_dict = dict()
 for key, values in ordered_pairs:
 if key in my_dict:
- raise ValueError("Duplicate key:{}".format(key,))
+ raise ValueError("Duplicate key:{}".format(key,))
 else:
- my_dict[key] = values
- return my_dict
+ my_dict[key] = values
+ return my_dict
diff --git a/06_execution_time.py b/06_execution_time.py
@@ -13,6 +13,7 @@
 
 
 import time
+import random
 
 
 class ExecutionTime:
@@ -25,9 +26,9 @@ def duration(self):
 
 # ---- run code ---- #
 
-import random
 
 timer = ExecutionTime()
 sample_list = list()
-my_list = [random.randint(1, 888898) for num in xrange(1, 1000000) if num % 2 == 0]
-print 'Finished in{} seconds.'.format(timer.duration())
+my_list = [random.randint(1, 888898) for num in
+ range(1, 1000000) if num % 2 == 0]
+print('Finished in{} seconds.'.format(timer.duration()))
diff --git a/07_benchmark_permissions_loading_django.py b/07_benchmark_permissions_loading_django.py
@@ -14,8 +14,8 @@ def timed(*args, **kw):
 te = time.time()
 all_times.append(te - ts)
 
- print all_times
- print numpy.mean(all_times)
+ print(all_times)
+ print(numpy.mean(all_times))
 return result
 
 return timed
@@ -39,4 +39,4 @@ def load_new_perms():
 while n < 10:
 create_new_db()
 load_new_perms()
- n += 1
+ n += 1
diff --git a/08_basic_email_web_crawler.py b/08_basic_email_web_crawler.py
@@ -1,6 +1,9 @@
 import requests
 import re
-import urlparse
+try:
+ from urllib.parse import urljoin
+except ImportError:
+ from urlparse import urljoin
 
 # regex
 email_re = re.compile(r'([\w\.,]+@[\w\.,]+\.\w+)')
@@ -20,13 +23,13 @@ def crawl(url):
 # Find links
 links = link_re.findall(req.text)
 
- print "\nFound{} links".format(len(links))
+ print("\nFound{} links".format(len(links)))
 
 # Search links for emails
 for link in links:
 
 # Get an absolute URL for a link
- link = urlparse.urljoin(url, link)
+ link = urljoin(url, link)
 
 # Find all emails on current page
 result.update(email_re.findall(req.text))
@@ -36,7 +39,7 @@ def crawl(url):
 if __name__ == '__main__':
 emails = crawl('http://www.realpython.com')
 
- print "\nScrapped e-mail addresses:"
+ print("\nScrapped e-mail addresses:")
 for email in emails:
- print email
- print "\n"
+ print(email)
+ print("\n")
diff --git a/09_basic_link_web_crawler.py b/09_basic_link_web_crawler.py
@@ -1,6 +1,9 @@
 import requests
 import re
-import urlparse
+try:
+ from urllib.parse import urljoin
+except ImportError:
+ from urlparse import urljoin
 
 # regex
 link_re = re.compile(r'href="(.*?)"')
@@ -17,17 +20,15 @@ def crawl(url):
 # Find links
 links = link_re.findall(req.text)
 
- print "\nFound{} links".format(len(links))
+ print("\nFound{} links".format(len(links)))
 
 # Search links for emails
 for link in links:
 
 # Get an absolute URL for a link
- link = urlparse.urljoin(url, link)
+ link = urljoin(url, link)
 
- print link
- 
+ print(link)
 
 if __name__ == '__main__':
 crawl('http://www.realpython.com')
-
diff --git a/10_find_files_recursively.py b/10_find_files_recursively.py
@@ -2,7 +2,7 @@
 import os
 
 # constants
-PATH = '/../../../..'
+PATH = './'
 PATTERN = '*.py'
 
 
@@ -14,18 +14,18 @@ def get_file_names(filepath, pattern):
 # matches.append(os.path.join(root, filename)) # full path
 matches.append(os.path.join(filename)) # just file name
 if matches:
- print "Found{} files:".format(len(matches))
+ print("Found{} files:".format(len(matches)))
 output_files(matches)
 else:
- print "No files found."
+ print("No files found.")
 else:
- print "Sorry that path does not exist. Try again."
+ print("Sorry that path does not exist. Try again.")
 
 
 def output_files(list_of_files):
 for filename in list_of_files:
- print filename
+ print(filename)
 
 
 if __name__ == '__main__':
- all_files = get_file_names(PATH, PATTERN)
+ all_files = get_file_names(PATH, PATTERN)
diff --git a/11_optimize_images_with_wand.py b/11_optimize_images_with_wand.py
@@ -1,9 +1,9 @@
 import fnmatch
 import os
 
-# sudo pip install Wand
+# pip install Wand
 from wand.image import Image
-# sudo pip install http://pypi.python.org/packages/source/h/hurry.filesize/hurry.filesize-0.9.tar.gz
+# pip install http://pypi.python.org/packages/source/h/hurry.filesize/hurry.filesize-0.9.tar.gz
 from hurry.filesize import size
 
 
@@ -19,12 +19,13 @@ def get_image_file_names(filepath, pattern):
 for filename in fnmatch.filter(filenames, pattern):
 matches.append(os.path.join(root, filename)) # full path
 if matches:
- print "Found{} files, with a total file size of{}.".format(len(matches), get_total_size(matches))
+ print("Found{} files, with a total file size of{}.".format(
+ len(matches), get_total_size(matches)))
 return matches
 else:
- print "No files found."
+ print("No files found.")
 else:
- print "Sorry that path does not exist. Try again."
+ print("Sorry that path does not exist. Try again.")
 
 
 def get_total_size(list_of_image_names):
@@ -35,15 +36,15 @@ def get_total_size(list_of_image_names):
 
 
 def resize_images(list_of_image_names):
- print "Optimizing ... "
+ print("Optimizing ... ")
 for index, image_name in enumerate(list_of_image_names):
 with open(image_name) as f:
 image_binary = f.read()
 with Image(blob=image_binary) as img:
 if img.height >= 600:
 img.transform(resize='x600')
 img.save(filename=image_name)
- print "Optimization complete."
+ print("Optimization complete.")
 
 
 if __name__ == '__main__':
diff --git a/12_csv_split.py b/12_csv_split.py
@@ -117,10 +117,10 @@ def parse_file(arguments):
 writer = writer.writerows(chunk)
 
 # Output info
- print ""
- print "Chunk #{}:".format(current_chunk)
- print "Filepath:{}".format(current_output)
- print "# of rows:{}".format(len(chunk))
+ print("")
+ print("Chunk #{}:".format(current_chunk))
+ print("Filepath:{}".format(current_output))
+ print("# of rows:{}".format(len(chunk)))
 
 # Create new chunk
 current_chunk += 1
diff --git a/13_random_name_generator.py b/13_random_name_generator.py
@@ -10,7 +10,7 @@ def random_name_generator(first, second, x):
 - number of random names
 """
 names = []
- for i in xrange(0, int(x)):
+ for i in range(0, int(x)):
 random_first = randint(0, len(first)-1)
 random_last = randint(0, len(second)-1)
 names.append("{0}{1}".format(
@@ -23,4 +23,4 @@ def random_name_generator(first, second, x):
 first_names = ["Drew", "Mike", "Landon", "Jeremy", "Tyler", "Tom", "Avery"]
 last_names = ["Smith", "Jones", "Brighton", "Taylor"]
 names = random_name_generator(first_names, last_names, 5)
-print '\n'.join(names)
+print('\n'.join(names))
diff --git a/15_check_my_environment.py b/15_check_my_environment.py
@@ -11,7 +11,7 @@ def __init__(self, configFile):
 pass
 
 def process(self):
- print "ok"
+ print("ok")
 
 if __name__ == "__main__":
 m = Main(some_script.CONFIGFILE)
@@ -39,7 +39,7 @@ def get_config_file():
 if CONFIGFILE is None:
 sys.exit("Configuration error! Unknown environment set. \
 Edit config.py and set appropriate environment")
-print "Config file:{}".format(CONFIGFILE)
+print("Config file:{}".format(CONFIGFILE))
 if not os.path.exists(CONFIGFILE):
 sys.exit("Configuration error! Config file does not exist")
-print "Config ok ...."
+print("Config ok ....")
diff --git a/18_zipper.py b/18_zipper.py
@@ -3,7 +3,7 @@
 from zipfile import ZipFile
 
 
-#set file name and time of creation
+# set file name and time of creation
 today = datetime.now()
 file_name = 'zipper_' + today.strftime('%Y.%m.%dh%H%M') + '.zip'
 dir_name = 'tmp/' # update path
diff --git a/20_restore_file_from_git.py b/20_restore_file_from_git.py
@@ -1,9 +1,9 @@
 from subprocess import check_output, call
 
 
-file_name = str(raw_input('Enter the file name: '))
+file_name = str(input('Enter the file name: '))
 commit = check_output(["git", "rev-list", "-n", "1", "HEAD", "--", file_name])
-print str(commit).rstrip()
+print(str(commit).rstrip())
 call(["git", "checkout", str(commit).rstrip()+"~1", file_name])
 
 
diff --git a/22_git_tag.py b/22_git_tag.py
@@ -10,5 +10,5 @@
 subprocess.call(command, shell=True)
 subprocess.call('git push --tags', shell=True)
 else:
- print 'usage: tag.py TAG_NAME COMMIT'
+ print('usage: tag.py TAG_NAME COMMIT')
 sys.exit(1)
diff --git a/24_sql2csv.py b/24_sql2csv.py
@@ -3,7 +3,7 @@
 import sqlite3
 
 if len(sys.argv) < 3:
- print "Use:{0} DATABASE_NAME TABLE_NAME".format(sys.argv[0])
+ print("Use:{0} DATABASE_NAME TABLE_NAME".format(sys.argv[0]))
 exit()
 
 conn = sqlite3.connect(sys.argv[1])
diff --git a/25_ip2geolocation.py b/25_ip2geolocation.py
diff --git a/26_stock_scraper.py b/26_stock_scraper.py
diff --git a/readme.md b/readme.md
diff --git a/requirements.txt b/requirements.txt

-Original file line number
+Diff line change
@@ @@ -1,3 +1,5 @@ @@
 .pyc
 .DS_Store
 -_tmp
 +_tmp
 +env
 +__pycache__
-Original file line number
+Diff line change
@@ @@ -1,18 +1,18 @@ @@
 -importurllib2
 +importrequests
 importre
 # get url
 -url=raw_input('Enter a URL (include `http://`): ')
 +url=input('Enter a URL (include `http://`): ')
 # connect to the url
 -website=urllib2.urlopen(url)
 +website=requests.get(url)
 # read html
 -html=website.read()
 +html=website.text
 # use re.findall to grab all the links
 links=re.findall('"((http|ftp)s?://.*?)"', html)
 # output links
 forlinkinlinks:
 -printlink[0]
 +print(link[0])
-Original file line number
+Diff line change
@@ @@ -1,25 +1,28 @@ @@
 importtwitter
+-
+-
++
++
 TWITTER_CONSUMER_KEY='XXX'
 TWITTER_CONSUMER_SECRET='XXX'
 TWITTER_ACCESS_TOKEN_KEY='XXX'
 TWITTER_ACCESS_TOKEN_SECRET='XXX'
+-
++
 twitter_api=twitter.Api(
 consumer_key=TWITTER_CONSUMER_KEY,
 consumer_secret=TWITTER_CONSUMER_SECRET,
 access_token_key=TWITTER_ACCESS_TOKEN_KEY,
 access_token_secret=TWITTER_ACCESS_TOKEN_SECRET
+)
+-
++
 if__name__=='__main__':
 follower_ids=twitter_api.GetFollowerIDs()
 following_ids=twitter_api.GetFriendIDs()
 -zombie_follows= [following_idforfollowing_idinfollowing_idsiffollowing_idnotinfollower_ids]
+-
 -confirm=raw_input("Are you sure you want to unfollow %s tweeps [y|n]? "% (len(zombie_follows)))
 +zombie_follows= [following_idforfollowing_idin
 +following_idsiffollowing_idnotinfollower_ids]
++
 +confirm=raw_input(
 +"Are you sure you want to unfollow{0} tweeps [y|n]? ".format(
 + (len(zombie_follows))))
 ifconfirm.lower() =='y':
 foridinzombie_follows:
 user=twitter_api.DestroyFriendship(user_id=id)
 -print"Unfollowed %s"%(user.screen_name)
 +print("Unfollowed {0}".format(user.screen_name))
-Original file line number
+Diff line change
@@ @@ -1,11 +1,9 @@ @@
 -importjson
+-
 defdict_raise_on_duplicates(ordered_pairs):
 """reject duplicate keys"""
 my_dict=dict()
 forkey, valuesinordered_pairs:
 ifkeyinmy_dict:
 -raiseValueError("Duplicate key:{}".format(key,))
 +raiseValueError("Duplicate key:{}".format(key,))
 else:
 -my_dict[key] =values
 -returnmy_dict
 +my_dict[key] =values
 +returnmy_dict
-Original file line number
+Diff line change
 importtime
 +importrandom
 classExecutionTime:
 # ---- run code ---- #
 -importrandom
 timer=ExecutionTime()
 sample_list=list()
 -my_list= [random.randint(1, 888898) fornuminxrange(1, 1000000) ifnum%2==0]
 -print'Finished in{} seconds.'.format(timer.duration())
 +my_list= [random.randint(1, 888898) fornumin
 +range(1, 1000000) ifnum%2==0]
 +print('Finished in{} seconds.'.format(timer.duration()))
-Original file line number
+Diff line change
 te=time.time()
 all_times.append(te-ts)
 -printall_times
 -printnumpy.mean(all_times)
 +print(all_times)
 +print(numpy.mean(all_times))
 returnresult
 returntimed
 whilen<10:
 create_new_db()
 load_new_perms()
 -n+=1
 +n+=1
-Original file line number
+Diff line change
 importos
 # constants
 -PATH='/../../../..'
 +PATH='./'
 PATTERN='*.py'
 # matches.append(os.path.join(root, filename)) # full path
 matches.append(os.path.join(filename)) # just file name
 ifmatches:
 -print"Found{} files:".format(len(matches))
 +print("Found{} files:".format(len(matches)))
 output_files(matches)
 else:
 -print"No files found."
 +print("No files found.")
 else:
 -print"Sorry that path does not exist. Try again."
 +print("Sorry that path does not exist. Try again.")
 defoutput_files(list_of_files):
 forfilenameinlist_of_files:
 -printfilename
 +print(filename)
 if__name__=='__main__':
 -all_files=get_file_names(PATH, PATTERN)
 +all_files=get_file_names(PATH, PATTERN)