Changeset 112c79 in git

Timestamp:

Aug 2, 2019, 7:35:10 PM (4 years ago)

Author:

Murray Heymann <heymann.murray@…>

Branches:

(u'spielwiese', '828514cf6e480e4bafc26df99217bf2a1ed1ef45')

Children:

58603fd7faac3d049f61ca0b65d89e4be0a664f9

Parents:

fece1392f8e9ff07b64d0ed4e5ec57bfa6dbf258

git-author:

Murray Heymann <heymann.murray@gmail.com>2019-08-02 19:35:10+02:00

git-committer:

Murray Heymann <heymann.murray@gmail.com>2019-08-02 19:35:13+02:00

Message:

Use sklearn to count keyword occurances

Location:

machine_learning

Files:

• common/keyword_vector.py (modified) (4 diffs)
• common/lookuptable.py (modified) (3 diffs)
• model/predictor.py (modified) (1 diff)
• predictor_runner.py (modified) (6 diffs)
• tests/common/test_lookuptable.py (modified) (2 diffs)
• tests/model/test_predictor.py (modified) (1 diff)

machine_learning/common/keyword_vector.py

@@ -7 +7 @@
 import sys
 import numpy as np
+from sklearn.feature_extraction.text import CountVectorizer
 
 from common.constants import KEYWORDS_FILE
@@ -19 +20 @@
         print("Please provide a valid input file as argument to read "
               "dictionary")
-        if sys.version_info[0] == 3:
+        if sys.version_info[0] == 3: # pylint: disable=no-else-raise
             raise FileNotFoundError
         else:
@@ -35 +36 @@
     return np.array(dictionary)
 
+def get_vectors(filenames, dictionary, normalise=True):
+    """
+    Create vectors from a dictionary and populate the counts according to
+    specified files
+    """
+    assert filenames is not None, \
+            "Please provide a valid list of files as argument"
+    assert not filenames.size == 0, \
+            "Please provide a valid list of files as argument"
+    for filename in filenames:
+        if not os.path.isfile(filename):
+            print("Please provide a valid input file as argument")
+            if sys.version_info[0] == 3: # pylint: disable=no-else-raise
+                raise FileNotFoundError
+            else:
+                print(filename)
+                raise IOError
+    assert dictionary is not None, \
+            "Please provide a valid dictionary as argument"
+    assert not dictionary.size == 0, \
+            "Please provide a valid dictionary as argument"
+
+    doc_strings = []
+    for filename in filenames:
+        doc_string = ""
+        with open(filename, "r+") as file:
+            line = file.readline()
+
+            while not line == "":
+                doc_string = doc_string + " " + line
+                line = file.readline()
+
+        doc_string = re.sub('[^0-9a-zA-Z\-\_]', ' ', doc_string) # pylint: disable=anomalous-backslash-in-string
+        doc_strings.append(doc_string)
+    doc_strings = np.array(doc_strings)
+
+    vectorizer = CountVectorizer(vocabulary=dictionary)
+    vectors = vectorizer.fit_transform(doc_strings)
+    vectors = vectors.toarray()
+    if normalise:
+        vectors = vectors / np.sqrt((vectors ** 2).sum(-1))[..., np.newaxis]
+    return vectors
+
 
 def count_occurances(filename, dictionary, normalise=True):
@@ -41 +85 @@
     a specified file
     """
-    if not os.path.isfile(filename):
-        print("Please provide a valid input file as argument")
-        if sys.version_info[0] == 3:
-            raise FileNotFoundError
-        else:
-            raise IOError
-    assert dictionary is not None, \
-            "Please provide a valid dictionary as argument"
-    assert not dictionary.size == 0, \
-            "Please provide a valid dictionary as argument"
-
-    vector = create_vector_dictionary(dictionary)
-    with open(filename, "r+") as file:
-        line = file.readline()
-
-        while not line == "":
-            words = re.sub('[^0-9a-zA-Z\-\_]', ' ', line).split() # pylint: disable=anomalous-backslash-in-string
-            for word in words:
-                if word in vector.keys():
-                    vector[word] = vector[word] + 1
-            line = file.readline()
-    vector = np.array(list(vector.values()))
-    if normalise:
-        vector = normalise_vector(vector)
-    return vector
+    res = get_vectors(np.array([filename]),
+                      dictionary,
+                      normalise=normalise)
+    return res[0]
 
 

machine_learning/common/lookuptable.py

@@ -14 +14 @@
 
 # local imports
-from common.keyword_vector import count_occurances, read_dictionary
+from common.keyword_vector import get_vectors, read_dictionary
 from common.constants import HELP_FILE_URL, HELP_FILE_PATH, SINGULAR_BIN, \
                         EXTRACT_SCRIPT, KEYWORDS_FILE, HELPFILE_NPY, \
@@ -82 +82 @@
         np.save(HELPFILE_NPY, file_list)
 
+        filenames = []
         for file in file_list:
-            vector = count_occurances(os.path.join(HELP_FILE_PATH, "html",
-                                                   file),
-                                      dictionary,
-                                      normalise=False)
-            vectors.append(vector)
-        vectors = np.array(vectors)
+            filename = os.path.join(HELP_FILE_PATH, "html", file)
+            filenames.append(filename)
+        filenames = np.array(filenames)
+        vectors = get_vectors(filenames, dictionary, normalise=False)
         np.save(VECTORS_NPY, vectors)
     else:
@@ -98 +97 @@
 
     return (vectors, file_list)
-
-

machine_learning/model/predictor.py

@@ -4 +4 @@
 
 # import cProfile
-import os
-import sys
-import time
 
 # Third party imports
 import numpy as np
 from sklearn.base import BaseEstimator, ClassifierMixin
-
-# Local imports
-from common.keyword_vector import vector_distance, count_occurances, \
-        read_dictionary, normalise_vector
-from common.lookuptable import create_table
-from common.constants import KEYWORDS_FILE
 
 

machine_learning/predictor_runner.py

@@ -5 +5 @@
 import os
 import sys
+import time
 import numpy as np
-from model.predictor import *
+
+from model.predictor import HelpPagePredictor
+from common.keyword_vector import read_dictionary, count_occurances
+from common.lookuptable import create_table
+from common.constants import KEYWORDS_FILE
 
 def find_prediction(filename):
+    """
+    Given a file name as string, get the predicted help page name
+    """
     dictionary = read_dictionary(KEYWORDS_FILE)
+
     start = time.time()
     vectors, file_list = create_table(dictionary=dictionary)
@@ -15 +24 @@
     print(end - start, "seconds to create_table")
 
+    return _find_prediction(filename, dictionary, vectors, file_list)
+
+
+def _find_prediction(filename, dictionary, vectors, file_list):
+    """
+    Train a predictor, get the predicted help page name
+    """
     predictor = HelpPagePredictor()
     predictor.fit(vectors, file_list)
@@ -24 +40 @@
     print(end - start, "seconds to make prediction")
     return prediction
-    
+
 
 def main():
@@ -42 +58 @@
     predictor.fit(vectors, file_list)
 
-    start = time.time()
-    test_vec = count_occurances("extract.lib", dictionary)
-    prediction = predictor.predict(np.array([test_vec]))
-    end = time.time()
-    print(end - start, "seconds to make prediction")
-    print(prediction)
-    print()
-
     print("prediction for zero vector")
     start = time.time()
@@ -59 +67 @@
     print()
 
+    prediction = _find_prediction("extract.lib",
+                                  dictionary,
+                                  vectors,
+                                  file_list)
+    print(prediction)
+    print()
+
+
     if len(sys.argv) >= 2:
         for i in range(len(sys.argv)):
@@ -65 +81 @@
             if not os.path.isfile(sys.argv[i]):
                 continue
+
             print("predicting for file", sys.argv[i])
-            start = time.time()
-            test_vec = count_occurances(sys.argv[i], dictionary)
-            prediction = predictor.predict(np.array([test_vec]))
-            end = time.time()
-            print(end - start, "seconds to make prediction")
+            prediction = _find_prediction(sys.argv[i],
+                                          dictionary,
+                                          vectors,
+                                          file_list)
             print(prediction)
             print()

machine_learning/tests/common/test_lookuptable.py

@@ -2 +2 @@
 import unittest
 import numpy as np
+import cProfile
 
 from common.lookuptable import *
+from common.keyword_vector import count_occurances
 from common.constants import KEYWORDS_FILE
 
@@ -32 +34 @@
 
 if __name__ == '__main__':
+    #cProfile.run("unittest.main()")
     unittest.main()

machine_learning/tests/model/test_predictor.py

@@ -5 +5 @@
 from model.predictor import *
 from common.constants import KEYWORDS_FILE
+
+from common.keyword_vector import normalise_vector, vector_distance
 
 class TestPredictionMethods(unittest.TestCase):