Changeset 91d4ae in git

Timestamp:

Jul 29, 2019, 2:45:44 PM (4 years ago)

Author:

Murray Heymann <heymann.murray@…>

Branches:

(u'spielwiese', '828514cf6e480e4bafc26df99217bf2a1ed1ef45')

Children:

eb2904af1f3fb2d653378807ed3b18780cfd4dbc

Parents:

d93ae5668f2da84265ee686cd190828895b49cba

Message:

Optimize methods

Location:

machine_learning

Files:

• common/keyword_vector.py (modified) (8 diffs)
• common/lookuptable.py (modified) (1 diff)
• model/predictor.py (modified) (5 diffs)

machine_learning/common/keyword_vector.py

@@ -7 +7 @@
 import re
 import sys
+import numpy as np
 
 ### Read from file ########################################################
@@ -40 +41 @@
         print("Please provide a valid dictionary as argument")
         return {}
-    vector = create_vector(dictionary)
+    if dictionary is None:
+        print("Please provide a valid dictionary as argument")
+        return {}
+    vector = create_vector_dictionary(dictionary)
     with open(filename, "r+") as file:
         line = file.readline()
 
         while not line == "":
-            # DONE: replace all non-alphanumeric characters with space
-            # words = line.split()
             words = re.sub('[^0-9a-zA-Z\-\_]', ' ', line).split() # pylint: disable=anomalous-backslash-in-string
             for word in words:
@@ -52 +54 @@
                     vector[word] = vector[word] + 1
             line = file.readline()
+    vector = np.array(list(vector.values()))
     if normalise:
-        normalise_vector(vector)
-    return vector
+        vector = normalise_vector(vector)
+    return vector 
 
 
@@ -73 +76 @@
     Return an identical copy of a vector
     """
-    new_vector = {}
-    for key in vector.keys():
-        new_vector[key] = vector[key]
-    return new_vector
+    return np.copy(np.array(vector))
 
 
 ### Vector specific logic #################################################
 
-def create_vector(dictionary):
+def create_vector_dictionary(dictionary):
     """
     Create a zero vector for a given dictionary
@@ -95 +95 @@
     Calculate the Euclidean distance between two vectors.
     """
-    if not set(vec1.keys()) == set(vec2.keys()):
-        print("Dictionaries don't have the same keys")
+    if not len(vec1) == len(vec2):
+        print("Vectors don't have the same sizes")
         return -1
 
-
-    dist = 0
-    for key in vec1:
-        dist = dist + (vec1[key] - vec2[key]) ** 2
-
-    dist = math.sqrt(dist)
+    diff_vec = vec1 - vec2
+    dist = np.linalg.norm(diff_vec)
 
     return dist
@@ -114 +110 @@
     distance of 1 between the zero vector and the vector itself.
     """
-    sum_vals = 0
-    for key in vec.keys():
-        sum_vals = sum_vals + (vec[key] * vec[key])
-
-    sum_vals = math.sqrt(sum_vals)
-
-    for key in vec.keys():
-        vec[key] = (vec[key] + 0.0) / sum_vals
+    vec = vec / np.linalg.norm(vec)
+    return vec
 
 
@@ -133 +123 @@
         sys.exit(1)
 
-    dic = read_dictionary(sys.argv[1])
-
-    testvector = {"hello":3, "bye":4}
+    testvector = np.array([3,4])
     normalise_vector(testvector)
     print("normalised vector: " + str(testvector))
 
-    vector1 = {"hello":3, "bye":4}
-    normalise_vector(vector1)
-    vector2 = {"hello":4, "bye":3}
+    vector1 = np.array([3,4])
+    vector1 = normalise_vector(vector1)
+    vector2 = np.array([4,3])
     normalise_vector(vector2)
+    vector2 = normalise_vector(vector2)
     print("distance same vector: " + str(vector_distance(vector1, vector1)))
     print("distance different vector: " + str(vector_distance(vector1, vector2)))
@@ -148 +137 @@
     print(vector2)
 
+    dic = read_dictionary(sys.argv[1])
     print(count_occurances("../Singular/table.h", dic))
 

machine_learning/common/lookuptable.py

@@ -76 +76 @@
         vectors = np.load(VECTORS_NPY)
         file_list = np.load(HELPFILE_NPY)
-    for vector in vectors:
-        normalise_vector(vector)
+
+    # normalise the vectors
+    vectors = vectors / np.sqrt((vectors ** 2).sum(-1))[..., np.newaxis]
 
     return (vectors, file_list)

machine_learning/model/predictor.py

@@ -12 +12 @@
 # Local imports
 from common.keyword_vector import vector_distance, count_occurances, \
-                                    read_dictionary
+        read_dictionary, normalise_vector
 from common.lookuptable import create_table
 from common.constants import KEYWORDS_FILE
@@ -48 +48 @@
         for x in X: # pylint: disable=invalid-name
             # find the closest vector
-
             min_val = float("inf")
-            min_vec = None
+            index = -1
+            i = 0
             for vec in self.vectors:
                 dist = vector_distance(x, vec)
                 if dist < min_val:
                     min_val = dist
-                    min_vec = vec
+                    index = i
+                i = i + 1
 
             # find corresponding filename
-            index = list(self.vectors).index(min_vec)
             file = self.files[index]
             ret_list.append(file)
@@ -70 +70 @@
     print("Running some tests")
     predictor = HelpPagePredictor()
-    vector1 = {"hello":1, "bye":4, "pizza": 10}
-    vector2 = {"hello":2, "bye":3, "pizza": 1}
-    vector3 = {"hello":3, "bye":9, "pizza": 3}
+    vector1 = normalise_vector([1, 4, 10])
+    vector2 = normalise_vector([2, 3, 1])
+    vector3 = normalise_vector([3, 9, 3])
 
     vectors = np.array([vector1, vector2, vector3])
@@ -78 +78 @@
     print(vectors)
     print(files)
+    print()
 
-    testvec = {"hello":1, "bye":1, "pizza": 1}
+    testvec = normalise_vector([1, 1, 1])
+    print("test vector:")
+    print(testvec)
+    print()
 
     print("distance to 1")
@@ -93 +97 @@
     predictor.fit(vectors, files)
     prediction = predictor.predict(np.array([testvec]))
+    print("Prediction:")
     print(prediction)
+    print()
 
     dictionary = read_dictionary(KEYWORDS_FILE)
+
     start = time.time()
     vectors, file_list = create_table(dictionary=dictionary)
     end = time.time()
     print(end - start, "seconds to create_table")
+
     test_vec = count_occurances("extract.lib", dictionary)
     predictor.fit(vectors, file_list)
+
     start = time.time()
     prediction = predictor.predict(np.array([test_vec]))
     end = time.time()
+    print(end - start, "seconds to make prediction")
     print(prediction)
-    print(end - start, "seconds to make prediction")
 
 if __name__ == '__main__':