Changeset 05dbb13 in git

Timestamp:

Jul 29, 2019, 5:43:56 PM (5 years ago)

Author:

Murray Heymann <heymann.murray@…>

Branches:

(u'spielwiese', '17f1d200f27c5bd38f5dfc6e8a0879242279d1d8')

Children:

c1b9ababa50e8059dd847da0957f50aea901e6a8

Parents:

9f0f63787f503527775273cff2d354bd558c9cda

Message:

Fix some errors, alphabetise keyword list

Location:

machine_learning

Files:

• common/keyword_vector.py (modified) (6 diffs)
• common/lookuptable.py (modified) (3 diffs)
• extract.lib (modified) (1 diff)
• model/predictor.py (modified) (1 diff)

machine_learning/common/keyword_vector.py

@@ -3 +3 @@
 """Some vector logic"""
 
+import numpy as np
 import os
 import re
 import sys
-import numpy as np
+import traceback
+
+from common.constants import KEYWORDS_FILE
 
 ### Read from file ########################################################
 
-def read_dictionary(filename):
+def read_dictionary(filename=KEYWORDS_FILE):
     """
     Read a dictionary saved as a textfile
@@ -26 +29 @@
             dictionary.append(line.strip())
             line = file.readline()
-    return dictionary
+    return np.array(dictionary)
 
 
@@ -37 +40 @@
         print("Please provide a valid input file as argument")
         return []
-    if dictionary == []:
+    if dictionary.size == 0:
         print("Please provide a valid dictionary as argument")
         return []
@@ -110 +113 @@
     distance of 1 between the zero vector and the vector itself.
     """
-    vec = vec / np.linalg.norm(vec)
+
+    if vec is None:
+        print("Please provide a valid vector")
+        print("Returning empty vector by default")
+        return np.array([])
+
+    if not isinstance(vec, np.ndarray):
+        print("Warning, vector should be a numpy array")
+    norm = np.linalg.norm(vec)
+    if not norm == 0:
+        vec = vec / norm
     return vec
 
@@ -118 +131 @@
     Run some basic tests
     """
-    if len(sys.argv) != 2:
-        print("Usage: ")
-        print(sys.argv[0] + " <dict_name>")
-        sys.exit(1)
 
     testvector = np.array([3, 4])
@@ -136 +145 @@
     print(vector1)
     print(vector2)
+    print()
 
-    dic = read_dictionary(sys.argv[1])
+    print("Attempt to normalise the zero vector")
+    print(normalise_vector(np.array([0,0,0,0,0])))
+    print()
+
+    print("Attempt to normalise list")
+    print(normalise_vector([3,4,0,0,0]))
+    print()
+
+    print("Attempt to normalise empty vector")
+    print(normalise_vector(np.array([])))
+    print()
+
+    print("Attempt to normalise None")
+    print(normalise_vector(None))
+    print()
+
+    if len(sys.argv) == 2:
+        dic = read_dictionary(filename=sys.argv[1])
+    else:
+        dic = read_dictionary()
+    print("vector of ../Singular/table.h")
     print(count_occurances("../Singular/table.h", dic))
 

machine_learning/common/lookuptable.py

@@ -49 +49 @@
     'keywords.txt'
     """
+    # extract keywords using the singular script
     os.system(SINGULAR_BIN + " " + EXTRACT_SCRIPT)
 
+    # read from the file created by singular
+    dictionary = read_dictionary()
+    print(dictionary)
 
-def create_table(dictionary=None):
+    # sort alphabetically
+    dictionary = np.sort(dictionary)
+    print(dictionary)
+
+    # write back to the same file
+    with open(KEYWORDS_FILE, "w") as file:
+        for word in dictionary:
+            file.write(word + "\n")
+
+
+
+def create_table(dictionary=None, attempt_cached=True):
     """
     Get a list of helpfiles, and generate a word occurance vector for each.
@@ -60 +75 @@
     vectors = []
 
-    if not os.path.isfile(VECTORS_NPY) or not os.path.isfile(HELPFILE_NPY):
+    if not os.path.isfile(VECTORS_NPY) or \
+            not os.path.isfile(HELPFILE_NPY) or \
+            not attempt_cached:
         file_list = np.array(get_list_of_htm_files())
         np.save(HELPFILE_NPY, file_list)
@@ -90 +107 @@
         print(file)
     extract_keywords()
-    vectors, files = create_table()
+    vectors, files = create_table(attempt_cached=False)
+    vectors1, files1 = create_table()
+
+    if not (vectors == vectors1).all():
+        print("Cached version differs from original version")
+    elif not (files == files1).all():
+        print("Cached version differs from original version")
+    else:
+        print("Cached version corresponds with original")
+
     dictionary = read_dictionary(KEYWORDS_FILE)
     test_vec = count_occurances(os.path.join(HELP_FILE_PATH, "html",
                                              files[1]), dictionary)
-    print((test_vec == vectors[1]).all())
+    print((test_vec==vectors[1]).all())
 
 

machine_learning/extract.lib

@@ -5 +5 @@
 int i;
 
+// combine lists, leaving out the first entry of l, which is "l"
+list combined = delete(l, 1) + k;
+
 // create file, overwrite if exists
-write(":w keywords.txt", l[1]);
+write(":w keywords.txt", combined[1]);
 
-// write entries from "names" procedure
-for (i = 1; i < size(l); i++) {
-        write(":a keywords.txt", l[i+1]);
-}
 
-// write entries from "reservedNameList" procedure
-for (i = 1; i < size(k) + 1; i++) {
-        write(":a keywords.txt", l[i]);
+// write remaining entries to file
+for (i = 2; i < size(combined) + 1; i++) {
+        write(":a keywords.txt", combined[i]);
 }
 

machine_learning/model/predictor.py

@@ -117 +117 @@
     print(prediction)
 
+
+    test_vec = count_occurances("test.txt", dictionary)
+    start = time.time()
+    prediction = predictor.predict(np.array([test_vec]))
+    end = time.time()
+    print(end - start, "seconds to make prediction")
+    print(prediction)
+
 if __name__ == '__main__':
     cProfile.run("main()")