Changeset f59883 in git

Timestamp:

Jul 30, 2019, 5:21:47 PM (5 years ago)

Author:

Murray Heymann <heymann.murray@…>

Branches:

(u'spielwiese', '17f1d200f27c5bd38f5dfc6e8a0879242279d1d8')

Children:

da892581a52069935f084604d05c0ecd6d19d5c9

Parents:

de88e1d84f18979811df4c3d1dc68f221a3362ef

Message:

Expand testing

Location:

machine_learning

Files:

• common/keyword_vector.py (modified) (9 diffs)
• model/predictor.py (modified) (3 diffs)

machine_learning/common/keyword_vector.py

@@ -5 +5 @@
 import os
 import re
-import sys
 import numpy as np
 
@@ -17 +16 @@
     """
     if not os.path.isfile(filename):
-        print("Please provide a valid input file as argument")
-        return np.array([])
+        print("Please provide a valid input file as argument to read "
+              "dictionary")
+        raise FileNotFoundError
 
     dictionary = []
@@ -38 +38 @@
     if not os.path.isfile(filename):
         print("Please provide a valid input file as argument")
-        return []
-    if dictionary.size == 0:
-        print("Please provide a valid dictionary as argument")
-        return []
-    if dictionary is None:
-        print("Please provide a valid dictionary as argument")
-        return []
+        raise FileNotFoundError
+    assert dictionary is not None, \
+            "Please provide a valid dictionary as argument"
+    assert not dictionary.size == 0, \
+            "Please provide a valid dictionary as argument"
+
     vector = create_vector_dictionary(dictionary)
     with open(filename, "r+") as file:
@@ -67 +66 @@
     Return an identical copy of a dictionary
     """
-    new_dic = []
-    for word in dictionary:
-        new_dic.append(word)
-    return new_dic
+    return np.copy(np.array(dictionary))
 
 
@@ -84 +80 @@
 def create_vector_dictionary(dictionary):
     """
-    Create a zero vector for a given dictionary
+    Create a zero lookup dictionary for a given dictionary
     """
     assert not dictionary is None, "Please give a dictionary"
@@ -98 +94 @@
     Calculate the Euclidean distance between two vectors.
     """
-    if not len(vec1) == len(vec2):
-        print("Vectors don't have the same sizes")
-        return -1
+    assert len(vec1) == len(vec2), \
+            "Vectors don't have the same sizes"
 
     dist = np.linalg.norm(vec1 - vec2)
@@ -114 +109 @@
 
     if vec is None:
-        print("Please provide a valid vector")
+        print("Warning, None is not a valid vector")
         print("Returning empty vector by default")
         return np.array([])
@@ -120 +115 @@
     if not isinstance(vec, np.ndarray):
         print("Warning, vector should be a numpy array")
+
+    if np.array(vec).size == 0:
+        print("Warning, vector being normalised is empty")
+
     norm = np.linalg.norm(vec)
     if not norm == 0:
@@ -125 +124 @@
     return vec
 
-
-def main():
-    """
-    Run some basic tests
-    """
-
-    testvector = np.array([3, 4])
-    normalise_vector(testvector)
-    print("normalised vector: " + str(testvector))
+def test_read_dictionary():
+    """
+    Create test for read_dictionary function
+    """
+    print("\033[1;32;40mTesting read_dictionary function:\033[1;37;40m")
+
+    print("Non-existant file")
+    correct = False
+    try:
+        read_dictionary("asdfasdf")
+    except FileNotFoundError:
+        print("correctly caught non-existant file")
+        print("\033[1;32;40mpass\033[1;37;40m")
+        correct = True
+    if not correct:
+        print("\033[1;31;40mfail\033[1;37;40m")
+
+    print("Reading default file")
+    correct = True
+    try:
+        read_dictionary()
+    except FileNotFoundError:
+        print("Default file for dictionary missing")
+        print("\033[1;31;40mfail\033[1;37;40m")
+        correct = False
+    if correct:
+        print("\033[1;32;40mpass\033[1;37;40m")
+    print()
+    print()
+
+def test_count_occurances():
+    """
+    Create test for count_occurances function
+    """
+    print("\033[1;32;40mTesting count_occurances function:\033[1;37;40m")
+    dic = read_dictionary()
+    correct = False
+    try:
+        vec = count_occurances("asdfasdf", dic)
+    except FileNotFoundError:
+        correct = True
+        print("Correctly raised FileNotFoundError")
+        print("\033[1;32;40mpass\033[1;37;40m")
+    if not correct:
+        print("\033[1;31;40mfail\033[1;37;40m")
+
+    print("Count occurances with None dictionary:")
+    correct = False
+    try:
+        count_occurances("../Singular/table.h", None)
+    except AssertionError:
+        print("Correctly caught AssertionError")
+        print("\033[1;32;40mpass\033[1;37;40m")
+        correct = True
+    if not correct:
+        print("\033[1;31;40mfail\033[1;37;40m")
+
+
+    print("Count occurances with empty dictionary:")
+    correct = False
+    try:
+        count_occurances("../Singular/table.h", np.array([]))
+    except AssertionError:
+        print("Correctly caught AssertionError")
+        print("\033[1;32;40mpass\033[1;37;40m")
+        correct = True
+    if not correct:
+        print("\033[1;31;40mfail\033[1;37;40m")
+
+
+    print("vector of ../Singular/table.h")
+    vec = count_occurances("../Singular/table.h", dic)
+    print(vec)
+    print()
+    print()
+
+def test_create_vector_dictionary():
+    """
+    Create test for create_vector_dictionary function
+    """
+    print("\033[1;32;40mTesting create_vector_dictionary " \
+            "function:\033[1;37;40m")
+    read_dictionary()
+
+    print("Create Vector Dictionary with None as dictionary:")
+    correct = False
+    try:
+        create_vector_dictionary(None)
+    except AssertionError:
+        correct = True
+        print("\033[1;32;40mpass\033[1;37;40m")
+    if not correct:
+        print("\033[1;31;40mfail\033[1;37;40m")
+
+    print("Create Vector Dictionary with empty dictionary:")
+    correct = False
+    try:
+        create_vector_dictionary(np.array([]))
+    except AssertionError:
+        correct = True
+        print("\033[1;32;40mpass\033[1;37;40m")
+    if not correct:
+        print("\033[1;31;40mfail\033[1;37;40m")
+
+    print()
+    print()
+
+def test_vector_distance():
+    """
+    Create test for vector_distance function
+    """
+    print("\033[1;32;40mTesting vector_distance function:\033[1;37;40m")
 
     vector1 = np.array([3, 4])
     vector1 = normalise_vector(vector1)
     vector2 = np.array([4, 3])
-    normalise_vector(vector2)
     vector2 = normalise_vector(vector2)
-    print("distance same vector: " + str(vector_distance(vector1, vector1)))
-    print("distance different vector: " + str(vector_distance(vector1, vector2)))
-    print(vector1)
-    print(vector2)
-    print()
+
+    print("Distance of vectors of different dimensions:")
+    correct = False
+    try:
+        vector_distance(np.array([1, 2, 3]), vector1)
+    except AssertionError:
+        correct = True
+        print("\033[1;32;40mpass\033[1;37;40m")
+    if not correct:
+        print("\033[1;31;40mfail\033[1;37;40m")
+
+
+    print("Distance same vector: " + str(vector_distance(vector1, vector1)))
+    assert vector_distance(vector1, vector1) == 0, \
+            "distance to same vectorshould be 0"
+    print("\033[1;32;40mpass\033[1;37;40m")
+
+    print("Distance different vector: " + str(vector_distance(vector1, vector2)))
+    assert vector_distance(vector1, vector2) > 0, \
+            "Distance between nonequal vectors should be strictly positive"
+    print("\033[1;32;40mpass\033[1;37;40m")
+    print()
+    print()
+
+def test_normalise_vector():
+    """
+    Create test for normalise_vector function
+    """
+    print("\033[1;32;40mTesting normalise_vector function:\033[1;37;40m")
+    testvector = np.array([3, 4])
+    testvector = normalise_vector(testvector)
+    assert np.linalg.norm(testvector) == 1, \
+            "Normalised vector should have norm of 1"
+    print("\033[1;32;40mpass\033[1;37;40m")
+    print("normalised vector: " + str(testvector))
+    print("\033[1;32;40mpass\033[1;37;40m")
 
     print("Attempt to normalise the zero vector")
     print(normalise_vector(np.array([0, 0, 0, 0, 0])))
-    print()
+    print("\033[1;32;40mpass\033[1;37;40m")
 
     print("Attempt to normalise list")
     print(normalise_vector([3, 4, 0, 0, 0]))
-    print()
+    print("\033[1;32;40mpass\033[1;37;40m")
 
     print("Attempt to normalise empty vector")
     print(normalise_vector(np.array([])))
-    print()
+    print("\033[1;32;40mpass\033[1;37;40m")
 
     print("Attempt to normalise None")
     print(normalise_vector(None))
-    print()
-
-    if len(sys.argv) == 2:
-        dic = read_dictionary(filename=sys.argv[1])
-    else:
-        dic = read_dictionary()
-    print("vector of ../Singular/table.h")
-    print(count_occurances("../Singular/table.h", dic))
+    print("\033[1;32;40mpass\033[1;37;40m")
+    print()
+    print()
+
+
+def main():
+    """
+    Run some basic tests
+    """
+    test_normalise_vector()
+
+    test_vector_distance()
+
+    test_read_dictionary()
+
+    test_count_occurances()
+
+    test_create_vector_dictionary()
 
 if __name__ == '__main__':

machine_learning/model/predictor.py

@@ -120 +120 @@
     print(end - start, "seconds to create_table")
 
-    test_vec = count_occurances("extract.lib", dictionary)
     predictor = HelpPagePredictor()
     predictor.fit(vectors, file_list)
 
     start = time.time()
+    test_vec = count_occurances("extract.lib", dictionary)
     prediction = predictor.predict(np.array([test_vec]))
     end = time.time()
@@ -132 +132 @@
 
     print("prediction for zero vector")
+    start = time.time()
     zerovec = np.zeros(len(dictionary) - 2)
-    start = time.time()
     prediction = predictor.predict(np.array([zerovec]))
     end = time.time()
@@ -147 +147 @@
                 continue
             print("predicting for file", sys.argv[i])
+            start = time.time()
             test_vec = count_occurances(sys.argv[i], dictionary)
-            start = time.time()
             prediction = predictor.predict(np.array([test_vec]))
             end = time.time()