* fixing memory management issues in callbacks

* adding a callback to allow external code to free the memory used to return the include callback's result
* API: un-exposing yara's memory management functions
* making include callbacks the only way ton include files
* adding a default include callback to open files from the disk (mimicking original behavior)
* updating docs accordingly

Author: edhoedt

docs/capi.rst

@@ -72,11 +72,12 @@ The callback receives the following parameters:
  * ``include_name``: name of the requested file.
  * ``calling_rule_filename``: the requesting file name (NULL if not a file).
  * ``calling_rule_namespace``: namespace (NULL if undefined).
- * ``user_data`` pointer is the same you passed to
- :c:func:`yr_compiler_set_include_callback`.
-It should return the requested file's content as a string. The memory for this string
-should be allocated by the callback function (yr_malloc can be used) but will
-be automatically freed by the yara compiler.
+ * ``user_data`` pointer is the same you passed to :c:func:`yr_compiler_set_include_callback`.
+It should return the requested file's content as a string. The memory for this
+string should be allocated by the callback function. Once it is safe to free the
+memory used to return the callback's result, the include_free function passed to
+:c:func:`yr_compiler_set_include_callback` will be called. If the memory does
+not need to be freed, NULL can be passed as include_free instead.
 
 The callback function has the following prototype:
 
@@ -88,6 +89,14 @@ The callback function has the following prototype:
       const char* calling_rule_namespace,
       void* user_data);
 
+The free function has the following prototype:
+
+.. code-block:: c
+
+  void include_free(
+      const char* callback_result_ptr,
+      void* user_data);
+
 After you successfully added some sources you can get the compiled rules
 using the :c:func:`yr_compiler_get_rules()` function. You'll get a pointer to
 a :c:type:`YR_RULES` structure which can be used to scan your data as
@@ -427,10 +436,13 @@ Functions
   pointer is passed to the callback function.
 
 
-.. c:function:: void yr_compiler_set_include_callback(YR_COMPILER* compiler, YR_COMPILER_INCLUDE_CALLBACK_FUNC callback, void* user_data)
+.. c:function:: void yr_compiler_set_include_callback(YR_COMPILER* compiler, YR_COMPILER_INCLUDE_CALLBACK_FUNC callback, YR_COMPILER_INCLUDE_FREE_FUNC include_free, void* user_data)
 
-  Set a callback to provide rules from a custom source when ``include`` directive
-  is invoked. The *user_data* pointer is passed to the callback function.
+  Set a callback to provide rules from a custom source when ``include``
+  directive is invoked. The *user_data* pointer is untouched and passed back to
+  the callback function and to the free function. Once the callback's result
+  is no longer needed, the include_free function will be called. If the memory
+  does not need to be freed, include_free can be set to NULL.
 
 
 .. c:function:: int yr_compiler_add_file(YR_COMPILER* compiler, FILE* file, const char* namespace, const char* file_name)
@@ -699,30 +711,6 @@ Functions
   Enables the specified rule. After being disabled with :c:func:`yr_rule_disable`
   a rule can be enabled again by using this function.
 
-.. c:function:: void* yr_calloc(size_t count, size_t size);
-
-  Cross-platform wrapper for HeapAlloc on Windows and calloc on other platforms.
-
-.. c:function:: void* yr_malloc(size_t size);
-
-  Cross-platform wrapper for HeapAlloc on Windows and malloc on other platforms.
-
-.. c:function:: void* yr_realloc(void* ptr, size_t size);
-
-  Cross-platform wrapper for HeapReAlloc on Windows and realloc on other platforms.
-
-.. c:function:: void yr_free(void* ptr);
-
-  Cross-platform wrapper for HeapFree on Windows and free on other platforms.
-
-.. c:function:: char* yr_strdup(const char *str);
-
-  Allocates a new buffer the same size as str and copies str to the new buffer.
-
-.. c:function:: char* yr_strdup(const char *str, size_t n);
-
-  Allocates a new buffer of size n and copies the n first character of str.
-
 
 Error codes
 -----------

libyara/compiler.c

@@ -56,12 +56,12 @@ YR_API int yr_compiler_create(
 
   new_compiler->errors = 0;
   new_compiler->callback = NULL;
-  new_compiler->include_callback = NULL;
+  new_compiler->include_callback = _yr_compiler_default_include_callback;
+  new_compiler->include_free = _yr_compiler_default_include_free;
   new_compiler->last_error = ERROR_SUCCESS;
   new_compiler->last_error_line = 0;
   new_compiler->current_line = 0;
   new_compiler->last_result = ERROR_SUCCESS;
-  new_compiler->file_stack_ptr = 0;
   new_compiler->file_name_stack_ptr = 0;
   new_compiler->fixup_stack_head = NULL;
   new_compiler->allow_includes = 1;
@@ -183,48 +183,116 @@ YR_API void yr_compiler_set_callback(
 }
 
 
-YR_API void yr_compiler_set_include_callback(
-    YR_COMPILER* compiler,
-    YR_COMPILER_INCLUDE_CALLBACK_FUNC include_callback,
-    void* user_data)
+const char* _yr_compiler_default_include_callback(
+  const char* include_name,
+  const char* calling_rule_filename,
+  const char* calling_rule_namespace,
+  void* user_data)
 {
-  compiler->include_callback = include_callback;
-  compiler->incl_clbk_user_data = user_data;
-}
-
-
-int _yr_compiler_push_file(
-    YR_COMPILER* compiler,
-    FILE* fh)
-{
-  if (compiler->file_stack_ptr < MAX_INCLUDE_DEPTH)
+  char* buffer;
+  //char  buffer[1024];
+  char* s = NULL;
+  #ifdef _WIN32
+  char* b = NULL;
+  #endif
+  char* f;
+  FILE* fh;
+  char* file_buffer;
+  size_t file_length;
+
+  if(calling_rule_filename != NULL)
   {
-    compiler->file_stack[compiler->file_stack_ptr] = fh;
-    compiler->file_stack_ptr++;
-    return ERROR_SUCCESS;
+    buffer = (char*) calling_rule_filename;
+    //strlcpy(buffer, calling_rule_filename, sizeof(buffer));
   }
   else
   {
-    compiler->last_result = ERROR_INCLUDE_DEPTH_EXCEEDED;
-    return ERROR_INCLUDE_DEPTH_EXCEEDED;
+    buffer = "\0";
+    //buffer[0] = '\0';
+  }
+  // make included file path relative to current source file
+  s = strrchr(buffer, '/');
+
+  #ifdef _WIN32
+  b = strrchr(buffer, '\\'); // in Windows both path delimiters are accepted
+  #endif
+  #ifdef _WIN32
+  if (s != NULL || b != NULL)
+  #else
+  if (s != NULL)
+  #endif
+  {
+    #ifdef _WIN32
+    f = (b > s)? (b + 1): (s + 1);
+    #else
+    f = s + 1;
+    #endif
+    strlcpy(f, include_name, sizeof(buffer) - (f - buffer));
+    f = buffer;
+    // SECURITY: Potential for directory traversal here.
+    fh = fopen(buffer, "r");
+    // if include file was not found relative to current source file,
+    // try to open it with path as specified by user (maybe user wrote
+    // a full path)
+    if (fh == NULL)
+    {
+      f = (char*) include_name;
+      // SECURITY: Potential for directory traversal here.
+      fh = fopen(include_name, "r");
+    }
+  }
+  else
+  {
+    f = (char*) include_name;
+    // SECURITY: Potential for directory traversal here.
+    fh = fopen(include_name, "r");
+  }
+  if (fh != NULL)
+  {
+    file_buffer = NULL;
+    file_length = 0;
+
+    fseek(fh, 0, SEEK_END);
+    file_length = ftell(fh);
+    fseek(fh, 0, SEEK_SET);
+    file_buffer = (char*) yr_malloc((file_length+1)*sizeof(char));
+    if(file_buffer)
+    {
+      if(file_length != fread(file_buffer, sizeof(char), file_length, fh))
+      {
+        return NULL;
+      }
+    }
+    fclose(fh);
+    return file_buffer;
   }
+  else return NULL;
 }
 
 
-FILE* _yr_compiler_pop_file(
-    YR_COMPILER* compiler)
+void _yr_compiler_default_include_free(
+  const char* callback_result_ptr,
+  void* user_data)
 {
-  FILE* result = NULL;
-
-  if (compiler->file_stack_ptr > 0)
+  if(callback_result_ptr != NULL)
   {
-    compiler->file_stack_ptr--;
-    result = compiler->file_stack[compiler->file_stack_ptr];
+    yr_free((void*)callback_result_ptr);
   }
+}
 
-  return result;
+
+YR_API void yr_compiler_set_include_callback(
+    YR_COMPILER* compiler,
+    YR_COMPILER_INCLUDE_CALLBACK_FUNC include_callback,
+    YR_COMPILER_INCLUDE_FREE_FUNC include_free,
+    void* user_data)
+{
+  compiler->include_callback = include_callback;
+  compiler->include_free = include_free;
+  compiler->incl_clbk_user_data = user_data;
 }
 
+
 int _yr_compiler_push_file_name(
     YR_COMPILER* compiler,
     const char* file_name)

libyara/include/yara/compiler.h

@@ -58,6 +58,10 @@ typedef const char* (*YR_COMPILER_INCLUDE_CALLBACK_FUNC)(
     const char* calling_rule_namespace,
     void* user_data);
 
+typedef void (*YR_COMPILER_INCLUDE_FREE_FUNC)(
+    const char* callback_result_ptr,
+    void* user_data );
+
 
 typedef struct _YR_FIXUP
 {
@@ -110,9 +114,6 @@ typedef struct _YR_COMPILER
   char*             file_name_stack[MAX_INCLUDE_DEPTH];
   int               file_name_stack_ptr;
 
-  FILE*             file_stack[MAX_INCLUDE_DEPTH];
-  int               file_stack_ptr;
-
   char              last_error_extra_info[MAX_COMPILER_ERROR_EXTRA_INFO];
 
   char              lex_buf[LEX_BUF_SIZE];
@@ -125,6 +126,8 @@ typedef struct _YR_COMPILER
 
   YR_COMPILER_CALLBACK_FUNC  callback;
   YR_COMPILER_INCLUDE_CALLBACK_FUNC include_callback;
+  YR_COMPILER_INCLUDE_FREE_FUNC include_free;
+
 
 } YR_COMPILER;
 
@@ -143,14 +146,6 @@ typedef struct _YR_COMPILER
         fmt, __VA_ARGS__);
 
 
-int _yr_compiler_push_file(
-    YR_COMPILER* compiler,
-    FILE* fh);
-
-
-FILE* _yr_compiler_pop_file(
-    YR_COMPILER* compiler);
-
 
 int _yr_compiler_push_file_name(
     YR_COMPILER* compiler,
@@ -160,6 +155,15 @@ int _yr_compiler_push_file_name(
 void _yr_compiler_pop_file_name(
     YR_COMPILER* compiler);
 
+const char* _yr_compiler_default_include_callback(
+    const char* include_name,
+    const char* calling_rule_filename,
+    const char* calling_rule_namespace,
+    void* user_data);
+
+void _yr_compiler_default_include_free(
+    const char* callback_result_ptr,
+    void* user_data);
 
 YR_API int yr_compiler_create(
     YR_COMPILER** compiler);
@@ -178,6 +182,7 @@ YR_API void yr_compiler_set_callback(
 YR_API void yr_compiler_set_include_callback(
     YR_COMPILER* compiler,
     YR_COMPILER_INCLUDE_CALLBACK_FUNC include_callback,
+    YR_COMPILER_INCLUDE_FREE_FUNC include_free,
     void* user_data);
 
 

libyara/include/yara/mem.h

@@ -46,24 +46,24 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #else
 
-YR_API void* yr_calloc(
+void* yr_calloc(
     size_t count,
     size_t size);
 
-YR_API void* yr_malloc(
+void* yr_malloc(
     size_t size);
 
-YR_API void* yr_realloc(
+void* yr_realloc(
     void* ptr,
     size_t size);
 
-YR_API void yr_free(
+void yr_free(
     void *ptr);
 
-YR_API char* yr_strdup(
+char* yr_strdup(
     const char *str);
 
-YR_API char* yr_strndup(
+char* yr_strndup(
 	const char *str, size_t n);
 
 #endif