Improved data/chart handling; added toy data

Author: wrichert

ch03/README.md

@@ -0,0 +1,7 @@
+Chapter 3 - Clustering - Finding Related Posts
+==============================================
+
+For this chapter you will need the '20news' dataset from
+http://mlcomp.org/datasets/379. To get the data you will need to
+register, but it is totally free. When being logged in, you will 
+see a ZIP download link.

ch03/data/toy/01.txt

@@ -0,0 +1 @@
+This is a toy post about machine learning. Actually, it contains not much interesting stuff.

ch03/data/toy/02.txt

@@ -0,0 +1 @@
+Imaging databases provide storage capabilities.

ch03/data/toy/03.txt

@@ -0,0 +1 @@
+Most imaging databases safe images permanently.

ch03/data/toy/04.txt

@@ -0,0 +1 @@
+Imaging databases store data.

ch03/data/toy/05.txt

@@ -0,0 +1 @@
+Imaging databases store data. Imaging databases store data. Imaging databases store data.

ch03/plot_kmeans_example.py

@@ -15,6 +15,8 @@
 frommatplotlibimportpylab
 fromsklearn.clusterimportKMeans
 
+fromutilsimportDATA_DIR, CHART_DIR
+
 seed=2
 sp.random.seed(seed) # to reproduce the data later on
 
@@ -55,7 +57,7 @@ def plot_clustering(x, y, title, mx=None, ymax=None, xmin=None, km=None):
 
 i=1
 plot_clustering(x, y, "Vectors")
-pylab.savefig(os.path.join("..", "1400_03_0%i.png"%i))
+pylab.savefig(os.path.join(CHART_DIR, "1400_03_0%i.png"%i))
 pylab.clf()
 
 i+=1
@@ -80,7 +82,7 @@ def plot_clustering(x, y, title, mx=None, ymax=None, xmin=None, km=None):
 c1a, c1b, c1c=km.cluster_centers_
 pylab.scatter(km.cluster_centers_[:, 0], km.cluster_centers_[:, 1],
 marker='x', linewidth=2, s=100, color='black')
-pylab.savefig(os.path.join("..", "1400_03_0%i.png"%i))
+pylab.savefig(os.path.join(CHART_DIR, "1400_03_0%i.png"%i))
 pylab.clf()
 
 i+=1
@@ -110,7 +112,7 @@ def plot_clustering(x, y, title, mx=None, ymax=None, xmin=None, km=None):
 pylab.gca().add_patch(
 pylab.Arrow(c1c[0], c1c[1], c2c[0] -c1c[0], c2c[1] -c1c[1], width=0.1))
 
-pylab.savefig(os.path.join("..", "1400_03_0%i.png"%i))
+pylab.savefig(os.path.join(CHART_DIR, "1400_03_0%i.png"%i))
 pylab.clf()
 
 i+=1
@@ -131,7 +133,7 @@ def plot_clustering(x, y, title, mx=None, ymax=None, xmin=None, km=None):
 
 pylab.scatter(km.cluster_centers_[:, 0], km.cluster_centers_[:, 1],
 marker='x', linewidth=2, s=100, color='black')
-pylab.savefig(os.path.join("..", "1400_03_0%i.png"%i))
+pylab.savefig(os.path.join(CHART_DIR, "1400_03_0%i.png"%i))
 pylab.clf()
 
 i+=1

ch03/rel_post_01.py

@@ -12,8 +12,10 @@
 
 fromsklearn.feature_extraction.textimportCountVectorizer
 
-DIR=r"../data/toy"
-posts= [open(os.path.join(DIR, f)).read() forfinos.listdir(DIR)]
+fromutilsimportDATA_DIR
+
+TOY_DIR=os.path.join(DATA_DIR, "toy")
+posts= [open(os.path.join(TOY_DIR, f)).read() forfinos.listdir(TOY_DIR)]
 
 new_post="imaging databases"
 

ch03/rel_post_mlcomp_01.py

@@ -5,22 +5,31 @@
 #
 # It is made available under the MIT License
 
+importos
+importsys
 importsklearn.datasets
 importscipyassp
 
+fromutilsimportDATA_DIR
+
+ifnotos.path.exists(DATA_DIR):
+print("""\
+It seems that you have not yet downloaded the MLCOMP data set.
+Please do so and place it into %s."""%DATA_DIR)
+sys.exit(1)
+
 new_post= \
 """Disk drive problems. Hi, I have a problem with my hard disk.
 After 1 year it is working only sporadically now.
 I tried to format it, but now it doesn't boot any more.
 Any ideas? Thanks.
 """
 
-MLCOMP_DIR=r"P:\Dropbox\pymlbook\data"
 groups= [
 'comp.graphics', 'comp.os.ms-windows.misc', 'comp.sys.ibm.pc.hardware',
 'comp.sys.ma c.hardware', 'comp.windows.x', 'sci.space']
 dataset=sklearn.datasets.load_mlcomp("20news-18828", "train",
-mlcomp_root=MLCOMP_DIR,
+mlcomp_root=DATA_DIR,
 categories=groups)
 print("Number of posts:", len(dataset.filenames))
 
@@ -82,6 +91,13 @@ def build_analyzer(self):
 show_at_2=similar[len(similar) /2]
 show_at_3=similar[-1]
 
+print("=== #1 ===")
 print(show_at_1)
+print()
+
+print("=== #2 ===")
 print(show_at_2)
+print()
+
+print("=== #3 ===")
 print(show_at_3)

ch03/utils.py

@@ -0,0 +1,17 @@
+# This code is supporting material for the book
+# Building Machine Learning Systems with Python
+# by Willi Richert and Luis Pedro Coelho
+# published by PACKT Publishing
+#
+# It is made available under the MIT License
+
+importos
+importsys
+
+DATA_DIR=os.path.join(
+os.path.dirname(os.path.realpath(__file__)), "data")
+
+ifnotos.path.exists(DATA_DIR):
+print("Uh, we were expecting a data directory, which contains the toy data")
+sys.exit(1)
+