Changeset cd552e in git for machine_learning/ml_python/common/lookuptable.py

Timestamp:

Aug 12, 2019, 4:20:41 PM (5 years ago)

Author:

Murray Heymann <heymann.murray@…>

Branches:

(u'spielwiese', 'fe61d9c35bf7c61f2b6cbf1b56e25e2f08d536cc')

Children:

87d423b777b7762ac55fd9308018f3d7a12b9940

Parents:

b03e2f1e6cf3ff16ea956608cf7b560430904959

git-author:

Murray Heymann <heymann.murray@gmail.com>2019-08-12 16:20:41+02:00

git-committer:

Murray Heymann <heymann.murray@gmail.com>2019-08-12 16:20:52+02:00

Message:

Create dir in home for cache files for ml

File:

• machine_learning/ml_python/common/lookuptable.py (modified) (7 diffs)

machine_learning/ml_python/common/lookuptable.py

@@ -17 +17 @@
 from common.constants import HELP_FILE_URL, HELP_FILE_PATH, SINGULAR_BIN, \
                         EXTRACT_SCRIPT, KEYWORDS_FILE, HELPFILE_NPY, \
-                        VECTORS_NPY
+                        VECTORS_NPY, HOME_DIR
 
 
@@ -50 +50 @@
     'keywords.txt'
     """
+    # ensure the homedir exists
+    if not os.path.isdir(HOME_DIR):
+        os.makedirs(HOME_DIR)
+
     # extract keywords using the singular script
-    os.system(SINGULAR_BIN + " " + EXTRACT_SCRIPT)
+    os.system(SINGULAR_BIN + " -q " + EXTRACT_SCRIPT + 
+            " | sort | uniq > " + KEYWORDS_FILE)
 
     # read from the file created by singular
     dictionary = read_dictionary()
-
-    # sort alphabetically
-    dictionary = np.sort(np.unique(dictionary))
-
-    # write back to the same file
-    with open(KEYWORDS_FILE, "w") as file:
-        for word in dictionary:
-            file.write(word + "\n")
 
     return dictionary
@@ -71 +68 @@
     Get a list of helpfiles, and generate a word occurance vector for each.
     """
+
     if dictionary is None:
         dictionary = read_dictionary(KEYWORDS_FILE)
@@ -78 +76 @@
             not os.path.isfile(HELPFILE_NPY) or \
             not attempt_cached:
+        os.makedirs(HOME_DIR, exist_ok=True)
         file_list = np.array(get_list_of_htm_files())
         np.save(HELPFILE_NPY, file_list)
@@ -101 +100 @@
     check whether the various files exist, and create if necessary.
     """
+    if not os.path.isdir(HOME_DIR):
+        os.makedirs(HOME_DIR)
+
     # check for and download help files if necessary
     tbz2_path = os.path.join(HELP_FILE_PATH, "helpfiles.tbz2")
@@ -111 +113 @@
     else:
         dictionary = None
-
 
     if not os.path.isfile(VECTORS_NPY) or not os.path.isfile(HELPFILE_NPY):
@@ -128 +129 @@
     if not os.path.isfile(KEYWORDS_FILE):
         retvalue = False
-    if not os.path.isfile(VECTORS_NPY) or not os.path.isfile(HELPFILE_NPY):
+    if not os.path.isdir(HOME_DIR) or \
+            not os.path.isfile(VECTORS_NPY) or \
+            not os.path.isfile(HELPFILE_NPY):
         retvalue = False