Changeset 0696a3 in git

Timestamp:

Aug 7, 2019, 10:07:51 PM (4 years ago)

Author:

Murray Heymann <heymann.murray@…>

Branches:

(u'spielwiese', '828514cf6e480e4bafc26df99217bf2a1ed1ef45')

Children:

092e6882fed83e361d79576bc0aa7dc34ac19ad2

Parents:

86420e9c548fd4f1645ed4114c5f79fc3488a8a6

Message:

Implement C prediction function

Location:

machine_learning

Files:

• ml_python/common/lookuptable.py (modified) (2 diffs)
• ml_python/predictor_runner.py (modified) (4 diffs)
• mlpredict.c (modified) (8 diffs)
• mlpredict.h (modified) (1 diff)
• testml.c (modified) (2 diffs)

machine_learning/ml_python/common/lookuptable.py

@@ -99 +99 @@
 def init_table_on_system():
     """
-    check whether the various files exist, and create if necessary. 
+    check whether the various files exist, and create if necessary.
     """
     # check for and download help files if necessary
@@ -114 +114 @@
 
     if not os.path.isfile(VECTORS_NPY) or not os.path.isfile(HELPFILE_NPY):
-        vectors, file_list = create_table(dictionary=dictionary,
-                                          attempt_cached=False)
+        create_table(dictionary=dictionary,
+                     attempt_cached=False)
 
 def is_lookup_initialised():

machine_learning/ml_python/predictor_runner.py

@@ -24 +24 @@
     print(end - start, "seconds to create_table")
 
-    return _find_prediction(filename, dictionary, vectors, file_list)
+    start = time.time()
+    pred = get_prediction(filename, dictionary, vectors, file_list)
+    end = time.time()
+    print(end - start, "seconds to make prediction.")
+    return pred
 
-
-def _find_prediction(filename, dictionary, vectors, file_list):
+def get_prediction(filename, dictionary, vectors, file_list):
     """
     Train a predictor, get the predicted help page name
@@ -34 +37 @@
     predictor.fit(vectors, file_list)
 
-    start = time.time()
     test_vec = count_occurances(filename, dictionary)
     prediction = predictor.predict(np.array([test_vec]))
-    end = time.time()
-    print(end - start, "seconds to make prediction")
-    return prediction
+    return prediction[0]
 
 
@@ -67 +67 @@
     print()
 
-    prediction = _find_prediction("extract.lib",
-                                  dictionary,
-                                  vectors,
-                                  file_list)
+    prediction = get_prediction("extract.lib",
+                                dictionary,
+                                vectors,
+                                file_list)
     print(prediction)
     print()
@@ -83 +83 @@
 
             print("predicting for file", sys.argv[i])
-            prediction = _find_prediction(sys.argv[i],
-                                          dictionary,
-                                          vectors,
-                                          file_list)
+            prediction = get_prediction(sys.argv[i],
+                                        dictionary,
+                                        vectors,
+                                        file_list)
             print(prediction)
             print()

machine_learning/mlpredict.c

@@ -11 +11 @@
 #include "mlpredict.h"
 
-PyObject *call_python_function(char *module, char *func);
+/* Locally defined macros */
+#define LOOKUPTABLE "common.lookuptable"
+#define KEYWORD_VECTOR "common.keyword_vector"
+#define PRD_RUNNER "predictor_runner"
+#define IS_LOOKUP_INITIALISED "is_lookup_initialised"
+#define INIT_TABLE_ON_SYSTEM "init_table_on_system"
+#define GET_PREDICTION "get_prediction"
+#define READ_DICTIONARY "read_dictionary"
+#define CREATE_TABLE "create_table"
+
+/**** Local Function Decleartions ****************************************/
+
+PyObject *_call_python_function(char *module, char *func);
+PyObject *_call_python_function_args(char *module, char *func, PyObject *pArgs);
+int _get_dictionary();
+int _get_vectors_file_list();
+
+/**** Static Variables ***************************************************/
+
+static PyObject *pDictionary = NULL;
+static PyObject *pVectors = NULL;
+static PyObject *pFile_list = NULL;
+
+/**** Public Functions ***************************************************/
 
 /**
  * Check whether the helpfiles have been downloaded and the relevant
- * vectors have been calculated and saved.
+ * vectors have been calculated and saved. Furthermore, the local static
+ * variables must have been set with pointers to the relevant data
+ * structures in the python instance. 
  *
  * @return An integer:  1 if it has been intialised, 0 otherwise
@@ -29 +54 @@
         } else {
                 /* python system is initialised */
-                pValue = call_python_function(LOOKUPTABLE, IS_LOOKUP_INITIALISED);
+                pValue = _call_python_function(LOOKUPTABLE, IS_LOOKUP_INITIALISED);
                 /* is this a boolean? */
                 if (pValue != NULL && PyBool_Check(pValue)) {
@@ -46 +71 @@
                 }
         }
+
+        retvalue = retvalue && pDictionary;
+        retvalue = retvalue && pVectors;
+        retvalue = retvalue && pFile_list;
         return retvalue;
 }
@@ -60 +89 @@
 int ml_initialise()
 {
+        int retvalue = 1;
         PyObject *pValue = NULL;
 
@@ -65 +95 @@
                 Py_Initialize();
         }
-        pValue = call_python_function(LOOKUPTABLE, INIT_TABLE_ON_SYSTEM);
+        pValue = _call_python_function(LOOKUPTABLE, INIT_TABLE_ON_SYSTEM);
         if (pValue != NULL) {
                 Py_DECREF(pValue);
-                return 1;
-        } else {
-                return 0;
-        }
-
+        } else {
+                retvalue = 0;
+        }
+
+        retvalue = retvalue && _get_dictionary();
+        retvalue = retvalue && _get_vectors_file_list();
+
+        return retvalue;
 }
 
@@ -84 +117 @@
 {
         int retvalue = 1;
+
+        Py_XDECREF(pDictionary);
+        Py_XDECREF(pVectors);
+        Py_XDECREF(pFile_list);
+
         if (Py_IsInitialized()) {
                 Py_Finalize();
@@ -90 +128 @@
                 retvalue = 0;
         }
-        return retvalue;
-}
-
-
-
-/**
- * Local helper function to call a function that takes no arguments.
- * @param[in] module A string of the module name where the function is
- * found
- * @param[in] func A string giving the name of the function to call.
- *
- * @return the returned PyObject.
- */
-PyObject *call_python_function(char *module, char *func)
-{
-        PyObject *pName = NULL, *pModule = NULL, *pFunc = NULL;
-        PyObject *pValue = NULL;
-                
-        if (!Py_IsInitialized()) {
-                Py_Initialize();
-        }
-
-        /* import the module */
-        pName = PyString_FromString(module);
-        pModule = PyImport_Import(pName);
-        Py_DECREF(pName);
-
-        if (pModule != NULL){
-                /* Get the init function we want to call */
-                pFunc = PyObject_GetAttrString(pModule, func);
-                if (pFunc && PyCallable_Check(pFunc)) {
-                        /* Callable function. Good. In this case, doesn't take any
-                         * arguments*/
-                        pValue = PyObject_CallObject(pFunc, NULL);
-                        if (pValue == NULL) {
-                                printf("No return for function\n");
-                                PyErr_Print();
-                        }
-                } else {
-                        /* Somehow not executable. Clean up! */
-                        if(PyErr_Occurred()) {
-                                PyErr_Print();
-                        }
-                        fprintf(stderr,
-                                        "Cannot find function \"%s\"\n",
-                                        func);
-                }
-                Py_XDECREF(pFunc);
-                Py_DECREF(pModule);
-        } else {
-                PyErr_Print();
-                fprintf(stderr, "Failed to load \"%s\"\n", module);
-        }
-
-        return pValue;
+
+        return retvalue;
 }
 
@@ -166 +151 @@
                                            int *pred_len)
 {
-
-
-
-        return 1;
-}
-
-
+        PyObject *pFName = NULL, *pArgs = NULL;
+        PyObject *pValue = NULL; PyObject *pString = NULL;
+        int retvalue = 1;
+        int ret_string_len = 0;
+        
+        pFName = PyString_FromString(filename);
+        if (!pFName) {
+                fprintf(stderr, "This is weird\n");
+
+                retvalue = 0;
+        } else {
+                pArgs = PyTuple_New(4);
+                if (!pArgs) {
+                        fprintf(stderr, "This is also weird\n");
+                        retvalue = 0;
+                        Py_DECREF(pFName);
+                } else {
+                        /* pFName is handed over to the tuple, so not DECREF later */
+                        PyTuple_SetItem(pArgs, 0, pFName);
+                        /* Since each of the following is handed over, we need to
+                         * increase the reference, otherwise our static variable
+                         * pointers might be freed by the python interpreter. */
+                        PyTuple_SetItem(pArgs, 1, pDictionary);
+                        Py_INCREF(pDictionary);
+                        PyTuple_SetItem(pArgs, 2, pVectors);
+                        Py_INCREF(pVectors);
+                        PyTuple_SetItem(pArgs, 3, pFile_list);
+                        Py_INCREF(pFile_list);
+                }
+
+        }
+        if (pArgs) {
+                pValue = _call_python_function_args(PRD_RUNNER,
+                                                                                        GET_PREDICTION,
+                                                                                        pArgs);
+                Py_DECREF(pArgs);
+                if (!pValue) {
+                        retvalue = 0;
+                } else {
+                        pString = PyObject_Str(pValue);
+                        strncpy(prediction_buffer,
+                                        PyString_AsString(pString),
+                                        buffer_size - 1);
+                        ret_string_len = strlen(PyString_AsString(pString));
+                        if (ret_string_len >= buffer_size - 1) {
+                                prediction_buffer[buffer_size - 1] = '\0';
+                                *pred_len = buffer_size - 1;
+                        } else {
+                                *pred_len = ret_string_len;
+                        }
+
+                        Py_DECREF(pString);
+                        Py_DECREF(pValue);
+                }
+        }
+
+        return retvalue;
+}
+
+/**** Local Functions ****************************************************/
+
+/**
+ * Local helper function to call a function that takes no arguments.
+ * @param[in] module A string of the module name where the function is
+ * found
+ * @param[in] func A string giving the name of the function to call.
+ *
+ * @return the returned PyObject.
+ */
+PyObject *_call_python_function(char *module, char *func) {
+        PyObject *pArgs = PyTuple_New(0);
+        PyObject *retvalue = _call_python_function_args(module, func, pArgs);
+
+        Py_DECREF(pArgs);
+        return retvalue;
+}
+
+
+/**
+ * Local helper function to call a function that takes arguments.
+ * @param[in] module A string of the module name where the function is
+ * found
+ * @param[in] func A string giving the name of the function to call.
+ * @param[in] pArgs The arguments to be parsed to the funcion being called.
+ *
+ * @return the returned PyObject.
+ */
+PyObject *_call_python_function_args(char *module, char *func, PyObject *pArgs)
+{
+        PyObject *pName = NULL, *pModule = NULL, *pFunc = NULL;
+        PyObject *pValue = NULL;
+                
+        if (!Py_IsInitialized()) {
+                Py_Initialize();
+                printf("I don't like this\n");
+        }
+
+        /* import the module */
+        pName = PyString_FromString(module);
+        pModule = PyImport_Import(pName);
+        Py_DECREF(pName);
+
+        if (pModule != NULL){
+                /* Get the init function we want to call */
+                pFunc = PyObject_GetAttrString(pModule, func);
+                if (pFunc && PyCallable_Check(pFunc)) {
+                        /* Callable function. Good. Call with the args supplied,
+                         * assuming the arguments are correct for the function */
+                        pValue = PyObject_CallObject(pFunc, pArgs);
+                        if (pValue == NULL) {
+                                printf("No return for function\n");
+                                PyErr_Print();
+                        }
+                } else {
+                        /* Somehow not executable. Clean up! */
+                        if(PyErr_Occurred()) {
+                                PyErr_Print();
+                        }
+                        fprintf(stderr,
+                                        "Cannot find function \"%s\"\n",
+                                        func);
+                }
+                Py_XDECREF(pFunc);
+                Py_DECREF(pModule);
+        } else {
+                PyErr_Print();
+                fprintf(stderr, "Failed to load \"%s\"\n", module);
+        }
+
+        return pValue;
+}
+
+/**
+ * Get the PyObject of the list of keywords called dictionary and set it to
+ * the local static variable.  If already set, simply return without any
+ * action.
+ *
+ * @return 1 if successful, 0 if something goes wrong.
+ */
+int _get_dictionary()
+{
+        int retvalue = 1;
+        PyObject *pValue = NULL;
+
+        if (!pDictionary) {
+                pValue = _call_python_function(KEYWORD_VECTOR, READ_DICTIONARY);
+                if (!pValue) {
+                        retvalue = 0;
+                } else {
+                        pDictionary = pValue;
+                }
+        }
+        return retvalue;
+}
+
+/**
+ * Get the PyObject of the list of vectors called corresponding to each
+ * helper file and the list of helper files that could be in a prediction
+ * and set it to the local static variable. If already set, simply return
+ * without any action.
+ *
+ * @return 1 if successful, 0 if something goes wrong.
+ */
+int _get_vectors_file_list()
+{
+        int retvalue = 1;
+        PyObject *pValue = NULL;
+        PyObject *pVal1 = NULL, *pVal2 = NULL;
+
+        if (!pVectors || !pFile_list) {
+                /* Ensure *both* are free */
+                Py_XDECREF(pVectors);
+                Py_XDECREF(pFile_list);
+                pVectors = NULL;
+                pFile_list = NULL;
+                pValue = _call_python_function(LOOKUPTABLE, CREATE_TABLE);
+                if (!pValue) {
+                        retvalue = 0;
+                } else {
+                        pVal1 = PyTuple_GetItem(pValue, 0);
+                        pVal2 = PyTuple_GetItem(pValue, 1);
+                        /* Decreasing the reference to the tuple causes the content to
+                         * be freed.  To prevent that (leading to a segfault), we have
+                         * to manually increase the reference to the contents */
+                        Py_INCREF(pVal1);
+                        Py_INCREF(pVal2);
+                        Py_DECREF(pValue);
+                        pValue = NULL;
+                        if (pVal1 && pVal2) {
+                                pVectors = pVal1;
+                                pFile_list = pVal2;
+                        } else {
+                                Py_XDECREF(pVal1);
+                                Py_XDECREF(pVal2);
+                                retvalue = 0;
+                        }
+                }
+        }
+        
+        return retvalue;
+}

machine_learning/mlpredict.h

@@ -17 +17 @@
 #define HELP_FILE_URL   "ftp://jim.mathematik.uni-kl.de/pub/Math/Singular/src/4-1-2/doc.tbz2" 
 #define HELP_FILE_PATH  "helpfiles/singular"
-
-#define LOOKUPTABLE "common.lookuptable"
-#define IS_LOOKUP_INITIALISED "is_lookup_initialised"
-#define INIT_TABLE_ON_SYSTEM "init_table_on_system"
 
 /**

machine_learning/testml.c

@@ -6 +6 @@
 {
         int i = 0;
+        int length = 0;
+        char buffer10[10];
+        char buffer11[11];
+        char buffer12[12];
+        char buffer20[20];
         if (argc != 1) {
                 printf("Usage: %s\n", argv[0]);
@@ -12 +17 @@
 
         i = ml_is_initialised();
-        printf("Returnvalue for ml_is_initialised: %d\n", i);
+        printf("Returnvalue for ml_is_initialised: \t%d\n", i);
         Py_Initialize();
 
-        i = ml_is_initialised();
-        printf("Returnvalue for ml_is_initialised: %d\n", i);
-
         i = ml_initialise();
-        printf("Returnvalue for ml_initialise: %d\n", i);
+        printf("Returnvalue for ml_initialise: \t\t%d\n", i);
 
         i = ml_is_initialised();
-        printf("Returnvalue for ml_is_initialised: %d\n", i);
+        printf("Returnvalue for ml_is_initialised: \t%d\n", i);
+
+        i = ml_make_prediction("test.txt", 10, buffer10, &length);
+        printf("Returnvalue for ml_make_prediciton: \t%d\n", i);
+        printf("String for buffer size 10: \t%s\n", buffer10);
+
+        i = ml_make_prediction("test.txt", 11, buffer11, &length);
+        printf("Returnvalue for ml_make_prediciton: \t%d\n", i);
+        printf("String for buffer size 11: \t%s\n", buffer11);
+
+        i = ml_make_prediction("test.txt", 12, buffer12, &length);
+        printf("Returnvalue for ml_make_prediciton: \t%d\n", i);
+        printf("String for buffer size 12: \t%s\n", buffer12);
+
+        i = ml_make_prediction("test.txt", 20, buffer20, &length);
+        printf("Returnvalue for ml_make_prediciton: \t%d\n", i);
+        printf("String for buffer size 20: \t%s\n", buffer20);
 
         i = ml_finalise();
-        printf("Returnvalue for ml_finalise: %d\n", i);
+        printf("Returnvalue for ml_finalise: \t\t%d\n", i);
 
         return 0;