Callback on include

docs/capi.rst

@@ -63,6 +63,31 @@ contains the file name and line number where the error or warning occurs.
 you're using :c:func:`yr_compiler_add_string`. The ``user_data`` pointer is the
 same you passed to :c:func:`yr_compiler_set_callback`.
 
+By default, for rules containing references to other files
+(``include "filename.yara"``), yara will try to find those files on disk.
+However, if you want to fetch the imported rules from another source (eg: from a
+database or remote service), a callback function can be set with
+:c:func:`yr_compiler_set_include_callback`.
+The callback receives the following parameters:
+ * ``include_name``: name of the requested file.
+ * ``calling_rule_filename``: the requesting file name (NULL if not a file).
+ * ``calling_rule_namespace``: namespace (NULL if undefined).
+ * ``user_data`` pointer is the same you passed to
+ :c:func:`yr_compiler_set_include_callback`.
+It should return the requested file's content as a string. The memory for this string
+should be allocated by the callback function (yr_malloc can be used) but will
+be automatically freed by the yara compiler.
+
+The callback function has the following prototype:
+
+.. code-block:: c
+
+  const char* include_callback(
+      const char* include_name,
+      const char* calling_rule_filename,
+      const char* calling_rule_namespace,
+      void* user_data);
+
 After you successfully added some sources you can get the compiled rules
 using the :c:func:`yr_compiler_get_rules()` function. You'll get a pointer to
 a :c:type:`YR_RULES` structure which can be used to scan your data as
@@ -402,6 +427,12 @@ Functions
   pointer is passed to the callback function.
 
 
+.. c:function:: void yr_compiler_set_include_callback(YR_COMPILER* compiler, YR_COMPILER_INCLUDE_CALLBACK_FUNC callback, void* user_data)
+
+  Set a callback to provide rules from a custom source when ``include`` directive
+  is invoked. The *user_data* pointer is passed to the callback function.
+
+
 .. c:function:: int yr_compiler_add_file(YR_COMPILER* compiler, FILE* file, const char* namespace, const char* file_name)
 
   Compile rules from a *file*. Rules are put into the specified *namespace*,
@@ -668,6 +699,31 @@ Functions
   Enables the specified rule. After being disabled with :c:func:`yr_rule_disable`
   a rule can be enabled again by using this function.
 
+.. c:function:: void* yr_calloc(size_t count, size_t size);
+
+  Cross-platform wrapper for HeapAlloc on Windows and calloc on other platforms.
+
+.. c:function:: void* yr_malloc(size_t size);
+
+  Cross-platform wrapper for HeapAlloc on Windows and malloc on other platforms.
+
+.. c:function:: void* yr_realloc(void* ptr, size_t size);
+
+  Cross-platform wrapper for HeapReAlloc on Windows and realloc on other platforms.
+
+.. c:function:: void yr_free(void* ptr);
+
+  Cross-platform wrapper for HeapFree on Windows and free on other platforms.
+
+.. c:function:: char* yr_strdup(const char *str);
+
+  Allocates a new buffer the same size as str and copies str to the new buffer.
+
+.. c:function:: char* yr_strdup(const char *str, size_t n);
+
+  Allocates a new buffer of size n and copies the n first character of str.
+
+
 Error codes
 -----------
 

docs/yarapython.rst

@@ -74,6 +74,36 @@ should be accepted in the source files, for example:
 If the source file contains include directives the previous line would raise
 an exception.
 
+If includes are used, a python callback can be set to define a custom source for
+the imported files (by default they are read from disk). This callback function
+is set through the ``include_callback`` optional parameter.
+It receives the following parameters:
+ *``requested_filename``: file requested with 'include'
+ *``filename``: file containing the 'include' directive if applicable, else None
+ *``namespace``: namespace
+And returns the requested rules sources as a single string.
+
+.. code-block:: python
+  import yara
+  import sys
+  if sys.version_info >= (3, 0):
+      import urllib.request as urllib
+  else:
+      import urllib as urllib
+
+  def mycallback(requested_filename, filename, namespace):
+      if requested_filename == 'req.yara':
+          uf = urllib.urlopen('https://pastebin.com/raw/siZ2sMTM')
+          sources = uf.read()
+          if sys.version_info >= (3, 0):
+              sources = str(sources, 'utf-8')
+          return sources
+      else:
+          raise Exception(filename+": Can't fetch "+requested_filename)
+
+  rules = yara.compile(source='include "req.yara" rule r{ condition: true }',
+                      include_callback=mycallback)
+
 If you are using external variables in your rules you must define those
 external variables either while compiling the rules, or while applying the
 rules to some file. To define your variables at the moment of compilation you

libyara/compiler.c

@@ -56,6 +56,7 @@ YR_API int yr_compiler_create(
 
   new_compiler->errors = 0;
   new_compiler->callback = NULL;
+  new_compiler->include_callback = NULL;
   new_compiler->last_error = ERROR_SUCCESS;
   new_compiler->last_error_line = 0;
   new_compiler->current_line = 0;
@@ -182,6 +183,16 @@ YR_API void yr_compiler_set_callback(
 }
 
 
+YR_API void yr_compiler_set_include_callback(
+    YR_COMPILER* compiler,
+    YR_COMPILER_INCLUDE_CALLBACK_FUNC include_callback,
+    void* user_data)
+{
+  compiler->include_callback = include_callback;
+  compiler->incl_clbk_user_data = user_data;
+}
+
+
 int _yr_compiler_push_file(
     YR_COMPILER* compiler,
     FILE* fh)

libyara/include/yara.h

@@ -39,5 +39,6 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "yara/error.h"
 #include "yara/stream.h"
 #include "yara/hash.h"
+#include "yara/mem.h"
 
 #endif

libyara/include/yara/compiler.h

@@ -52,6 +52,13 @@ typedef void (*YR_COMPILER_CALLBACK_FUNC)(
     void* user_data);
 
 
+typedef const char* (*YR_COMPILER_INCLUDE_CALLBACK_FUNC)(
+    const char* include_name,
+    const char* calling_rule_filename,
+    const char* calling_rule_namespace,
+    void* user_data);
+
+
 typedef struct _YR_FIXUP
 {
   void* address;
@@ -114,8 +121,10 @@ typedef struct _YR_COMPILER
 
   char              include_base_dir[MAX_PATH];
   void*             user_data;
+  void*             incl_clbk_user_data;
 
   YR_COMPILER_CALLBACK_FUNC  callback;
+  YR_COMPILER_INCLUDE_CALLBACK_FUNC include_callback;
 
 } YR_COMPILER;
 
@@ -166,6 +175,12 @@ YR_API void yr_compiler_set_callback(
     void* user_data);
 
 
+YR_API void yr_compiler_set_include_callback(
+    YR_COMPILER* compiler,
+    YR_COMPILER_INCLUDE_CALLBACK_FUNC include_callback,
+    void* user_data);
+
+
 YR_API int yr_compiler_add_file(
     YR_COMPILER* compiler,
     FILE* rules_file,

libyara/include/yara/mem.h

@@ -31,6 +31,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #define YR_MEM_H
 
 #include <stdio.h>
+#include <yara/utils.h>
 
 #ifdef DMALLOC
 
@@ -45,24 +46,24 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #else
 
-void* yr_calloc(
+YR_API void* yr_calloc(
     size_t count,
     size_t size);
 
-void* yr_malloc(
+YR_API void* yr_malloc(
     size_t size);
 
-void* yr_realloc(
+YR_API void* yr_realloc(
     void* ptr,
     size_t size);
 
-void yr_free(
+YR_API void yr_free(
     void *ptr);
 
-char* yr_strdup(
+YR_API char* yr_strdup(
     const char *str);
 
-char* yr_strndup(
+YR_API char* yr_strndup(
 	const char *str, size_t n);
 
 #endif