Changeset cd552e in git

Timestamp:

Aug 12, 2019, 4:20:41 PM (4 years ago)

Author:

Murray Heymann <heymann.murray@…>

Branches:

(u'spielwiese', 'd1ba061a762c62d3a25159d8da8b6e17332291fa')

Children:

87d423b777b7762ac55fd9308018f3d7a12b9940

Parents:

b03e2f1e6cf3ff16ea956608cf7b560430904959

git-author:

Murray Heymann <heymann.murray@gmail.com>2019-08-12 16:20:41+02:00

git-committer:

Murray Heymann <heymann.murray@gmail.com>2019-08-12 16:20:52+02:00

Message:

Create dir in home for cache files for ml

Files:

• Singular/iparith.cc (modified) (1 diff)
• machine_learning/extract.lib (modified) (1 diff)
• machine_learning/ml_python/common/constants.py (modified) (1 diff)
• machine_learning/ml_python/common/keyword_vector.py (modified) (2 diffs)
• machine_learning/ml_python/common/lookuptable.py (modified) (7 diffs)
• machine_learning/ml_python/tests/common/test_keyword_vectors.py (modified) (2 diffs)
• machine_learning/ml_python/tests/common/test_lookuptable.py (modified) (4 diffs)
• machine_learning/mlpredict.c (modified) (3 diffs)
• machine_learning/mlpredict.h (modified) (3 diffs)

Singular/iparith.cc

@@ -7948 +7948 @@
         unsigned nCount = (sArithBase.nCmdUsed-1) / 3;
 
-        if (ml_is_initialised()) {
-                printf("ml is initialised\n");
-        } else {
-                printf("ml is NOT initialised\n");
-        }
+        ml_initialise();
+        ml_finalise();
 
         if ((3*nCount) < sArithBase.nCmdUsed) {

machine_learning/extract.lib

@@ -8 +8 @@
 list combined = delete(l, 1) + k;
 
-// create file, overwrite if exists
-write(":w keywords.txt", combined[1]);
-
-
-// write remaining entries to file
-for (i = 2; i < size(combined) + 1; i++) {
-        write(":a keywords.txt", combined[i]);
+// printentries to STDOUT
+for (i = 1; i < size(combined) + 1; i++) {
+        print(combined[i]);
 }
 

machine_learning/ml_python/common/constants.py

@@ -4 +4 @@
 import os
 
-SINGULAR_BIN = "~/Singular/Singular4/bin/Singular"
+HOME_DIR       = os.path.expanduser("~/.singular")
+SINGULAR_BIN   = os.path.expanduser("~/Singular/Singular4/bin/Singular")
 EXTRACT_SCRIPT = "extract.lib"
-KEYWORDS_FILE = "keywords.txt"
-VECTORS_NPY = ".vectors.npy"
-HELPFILE_NPY = ".helpfilelist.npy"
+KEYWORDS_FILE  = os.path.expanduser("~/.singular/keywords.txt")
+VECTORS_NPY    = os.path.expanduser("~/.singular/vectors.npy")
+HELPFILE_NPY   = os.path.expanduser("~/.singular/helpfilelist.npy")
 
-HELP_FILE_URL = "ftp://jim.mathematik.uni-kl.de/pub/Math/Singular/src/4-1-2/doc.tbz2"
-HELP_FILE_PATH = os.path.join("helpfiles", "singular")
+HELP_FILE_URL  = "ftp://jim.mathematik.uni-kl.de/pub/Math/Singular/src/4-1-2/doc.tbz2"
+HELP_FILE_PATH = os.path.join(os.path.expanduser("~/.singular"), 
+        "helpfiles", "singular")

machine_learning/ml_python/common/keyword_vector.py

@@ -20 +20 @@
         print("Please provide a valid input file as argument to read "
               "dictionary")
-        if sys.version_info[0] == 3: # pylint: disable=no-else-raise
-            raise FileNotFoundError
-        else:
-            raise IOError
+        print(filename)
+        raise IOError
 
 
@@ -48 +46 @@
         if not os.path.isfile(filename):
             print("Please provide a valid input file as argument")
-            if sys.version_info[0] == 3: # pylint: disable=no-else-raise
-                raise FileNotFoundError
-            else:
-                print(filename)
-                raise IOError
+            print(filename)
+            raise IOError
     assert dictionary is not None, \
             "Please provide a valid dictionary as argument"

machine_learning/ml_python/common/lookuptable.py

@@ -17 +17 @@
 from common.constants import HELP_FILE_URL, HELP_FILE_PATH, SINGULAR_BIN, \
                         EXTRACT_SCRIPT, KEYWORDS_FILE, HELPFILE_NPY, \
-                        VECTORS_NPY
+                        VECTORS_NPY, HOME_DIR
 
 
@@ -50 +50 @@
     'keywords.txt'
     """
+    # ensure the homedir exists
+    if not os.path.isdir(HOME_DIR):
+        os.makedirs(HOME_DIR)
+
     # extract keywords using the singular script
-    os.system(SINGULAR_BIN + " " + EXTRACT_SCRIPT)
+    os.system(SINGULAR_BIN + " -q " + EXTRACT_SCRIPT + 
+            " | sort | uniq > " + KEYWORDS_FILE)
 
     # read from the file created by singular
     dictionary = read_dictionary()
-
-    # sort alphabetically
-    dictionary = np.sort(np.unique(dictionary))
-
-    # write back to the same file
-    with open(KEYWORDS_FILE, "w") as file:
-        for word in dictionary:
-            file.write(word + "\n")
 
     return dictionary
@@ -71 +68 @@
     Get a list of helpfiles, and generate a word occurance vector for each.
     """
+
     if dictionary is None:
         dictionary = read_dictionary(KEYWORDS_FILE)
@@ -78 +76 @@
             not os.path.isfile(HELPFILE_NPY) or \
             not attempt_cached:
+        os.makedirs(HOME_DIR, exist_ok=True)
         file_list = np.array(get_list_of_htm_files())
         np.save(HELPFILE_NPY, file_list)
@@ -101 +100 @@
     check whether the various files exist, and create if necessary.
     """
+    if not os.path.isdir(HOME_DIR):
+        os.makedirs(HOME_DIR)
+
     # check for and download help files if necessary
     tbz2_path = os.path.join(HELP_FILE_PATH, "helpfiles.tbz2")
@@ -111 +113 @@
     else:
         dictionary = None
-
 
     if not os.path.isfile(VECTORS_NPY) or not os.path.isfile(HELPFILE_NPY):
@@ -128 +129 @@
     if not os.path.isfile(KEYWORDS_FILE):
         retvalue = False
-    if not os.path.isfile(VECTORS_NPY) or not os.path.isfile(HELPFILE_NPY):
+    if not os.path.isdir(HOME_DIR) or \
+            not os.path.isfile(VECTORS_NPY) or \
+            not os.path.isfile(HELPFILE_NPY):
         retvalue = False
 

machine_learning/ml_python/tests/common/test_keyword_vectors.py

@@ -15 +15 @@
         print("Non-existant file")
         if sys.version_info[0] == 3:
-            self.assertRaises(FileNotFoundError,
+            #self.assertRaises(FileNotFoundError,
+            self.assertRaises(IOError,
                               read_dictionary,
                               "asdfasdf")
@@ -38 +39 @@
         print("Test non-existant file")
         if sys.version_info[0] == 3:
-            self.assertRaises(FileNotFoundError,
+            #self.assertRaises(FileNotFoundError,
+            self.assertRaises(IOError,
                               count_occurances,
                               "asdfasdf",

machine_learning/ml_python/tests/common/test_lookuptable.py

@@ -6 +6 @@
 from common.lookuptable import *
 from common.keyword_vector import count_occurances
-from common.constants import KEYWORDS_FILE, VECTORS_NPY, HELP_FILE_PATH
+from common.constants import KEYWORDS_FILE, VECTORS_NPY, HELP_FILE_PATH, \
+        HOME_DIR
 
 class TestLookuptableMethods(unittest.TestCase):
@@ -13 +14 @@
         os.system("rm -r " + HELP_FILE_PATH)
         fetch_tbz2_data()
-        fetch_tbz2_data()
         files = get_list_of_htm_files()
         self.assertGreater(len(files), 0)
 
     def test_extract_keywords(self):
-        extract_keywords()
+        #extract_keywords()
         self.assertTrue(os.path.isfile(KEYWORDS_FILE))
 
@@ -35 +35 @@
     def test_init_table_on_system(self):
         tbz2_path = os.path.join(HELP_FILE_PATH, "helpfiles.tbz2")
-        os.remove(tbz2_path)
-        os.remove(KEYWORDS_FILE)
-        os.remove(VECTORS_NPY)
+        os.system("rm -r " + HOME_DIR)
 
         init_table_on_system()
@@ -66 +64 @@
 
 if __name__ == '__main__':
+    if os.path.isdir(HOME_DIR):
+        os.system("rm -r " + HOME_DIR)
+    extract_keywords()
+    fetch_tbz2_data()
     #cProfile.run("unittest.main()")
     unittest.main()

machine_learning/mlpredict.c

@@ -8 +8 @@
 #include <stdio.h>
 #include <Python.h>
+
+#include "mlpredict.h"
 
 /* Locally defined macros */
@@ -107 +109 @@
 
 /**
- * A wrapper for Py_Finalize, checking whether it is necessary in the first
- * place.
+ * Tell python to decrement the global variables, checking whether it is
+ * necessary in the first place.
  *
  * @return An integer: 1 if successful, 0 if not.
@@ -116 +118 @@
         int retvalue = 1;
 
-        Py_XDECREF(pDictionary);
-        Py_XDECREF(pVectors);
-        Py_XDECREF(pFile_list);
-
         if (Py_IsInitialized()) {
-                Py_Finalize();
-                retvalue = 1;
+                Py_XDECREF(pDictionary);
+                Py_XDECREF(pVectors);
+                Py_XDECREF(pFile_list);
+                pDictionary = NULL;
+                pVectors = NULL;
+                pFile_list = NULL;
+
+                /* this breaks libpython2.7.so, so leave out: */
+                /* Py_Finalize(); */ 
         } else {
                 retvalue = 0;

machine_learning/mlpredict.h

@@ -8 +8 @@
 #ifndef MLPREDICT_H
 #define MLPREDICT_H
+
+#ifdef __cplusplus
 extern "C" {
+#endif /* ifdef cpp */
 
 /**
@@ -29 +32 @@
 
 /**
- * Finalize the python interpreter
+ * Tell python to decrement the global variables, checking whether it is
+ * necessary in the first place.
  *
  * @return An integer: 1 if successful, 0 if not.
@@ -54 +58 @@
                                            int *pred_len);
 
+#ifdef __cplusplus
 }
-#endif
+#endif /* ifdef cpp */
+#endif