feat: Mole uses Yara 4.x

Author: Xumeiquer

CHANGELOG.md

@@ -19,6 +19,5 @@
 ### Features
 - Added mole logger in engine package (0fc40f9)
 
-[Unreleased]: https://github.com/mole-ids/mole/compare/v0.1.2...HEAD
-[v0.1.2]: https://github.com/mole-ids/mole/compare/v0.1.1...v0.1.2
+[Unreleased]: https://github.com/mole-ids/mole/compare/v0.1.1...HEAD
 [v0.1.1]: https://github.com/mole-ids/mole/compare/v0.1.0...v0.1.1

docs/content/getting-started/install-mole.md

@@ -9,7 +9,7 @@ box. However, the Mole IDS team is working to port Mole to the major platforms.
 
 !!! note "Mole IDS Dependencies & Requirements"
     Mole IDS is build upon two libraries and they have to be installed on the
-    system you want to run Mole IDS. 
+    system you want to run Mole IDS.
 
     * [Yara](https://virustotal.github.io/yara/)
     * [PF_RING](https://www.ntop.org/products/packet-capture/pf_ring/)
@@ -85,13 +85,12 @@ echo pf_ring | sudo tee -a /etc/modules
 
 ### Install Yara
 
-At the moment Mole IDS uses Yara version 3.11.0. We know there is a newer version
-of Yara and we will added asoon as possible.
+Mole IDS uses the latest Yara version avaliable at the moment, which is Yara v4.0.2.
 
 ```shell
-wget https://github.com/VirusTotal/yara/archive/v3.11.0.tar.gz -O yara.tgz
+wget https://github.com/VirusTotal/yara/archive/4.0.2.tar.gz -O yara.tgz
 tar xvfz yara.tgz
-cd yara-3.11.0
+cd yara-4.0.2
 ./bootstrap.sh
 ./configure --enable-magic
 make

docs/content/writing-rules.md

@@ -1,7 +1,7 @@
 # Writting rules
 
 Mole IDS rule system is built on top of yara. You can find information about
-[writting basic yara rules](https://yara.readthedocs.io/en/v3.11.0/writingrules.html)
+[writting basic yara rules](https://yara.readthedocs.io/en/stable/writingrules.html)
 
 ## Syntax
 

go.mod

@@ -3,16 +3,14 @@ module github.com/mole-ids/mole
 go 1.14
 
 require (
-	github.com/google/gopacket v1.1.17
-	github.com/hillu/go-yara v1.2.2
-	github.com/k0kubun/pp v3.0.1+incompatible
-	github.com/mattn/go-colorable v0.1.6 // indirect
+	github.com/google/gopacket v1.1.18
+	github.com/hillu/go-yara/v4 v4.0.2
 	github.com/oklog/ulid v1.3.1
 	github.com/pkg/errors v0.9.1
 	github.com/spf13/cobra v1.0.0
 	github.com/spf13/pflag v1.0.5 // indirect
-	github.com/spf13/viper v1.6.3
-	go.uber.org/zap v1.14.1
-	golang.org/x/sys v0.0.0-20200420163511-1957bb5e6d1f // indirect
-	golang.org/x/tools v0.0.0-20200425043458-8463f397d07c // indirect
+	github.com/spf13/viper v1.7.0
+	go.uber.org/zap v1.15.0
+	golang.org/x/sys v0.0.0-20200728102440-3e129f6d46b1 // indirect
+	golang.org/x/tools v0.0.0-20200731060945-b5fad4ed8dd6 // indirect
 )

go.sum

@@ -14,7 +14,7 @@
 package tree
 
 import (
-	"github.com/hillu/go-yara"
+	"github.com/hillu/go-yara/v4"
 	"github.com/pkg/errors"
 
 	"github.com/mole-ids/mole/internal/nodes"

internal/tree/tree.go

@@ -14,7 +14,7 @@
 package types
 
 import (
-	"github.com/hillu/go-yara"
+	"github.com/hillu/go-yara/v4"
 	"github.com/mole-ids/mole/internal/nodes"
 )
 

internal/types/types.go

@@ -18,6 +18,7 @@ import (
 
 	"github.com/google/gopacket"
 	"github.com/google/gopacket/layers"
+	"github.com/hillu/go-yara/v4"
 	"github.com/pkg/errors"
 	"go.uber.org/zap"
 
@@ -172,7 +173,10 @@ func (motor *Engine) checkAndFire(pe *PacketExtractor) {
 
 	for _, matchID := range matches {
 		if scanner, found := motor.RuleMap[matchID]; found {
-			matches, err := scanner.ScanMem(pe.GetPacketPayload())
+			var matches yara.MatchRules
+			scanner = scanner.SetCallback(&matches)
+
+			err := scanner.ScanMem(pe.GetPacketPayload())
 			if err != nil {
 				logger.Log.Errorf(ScannerScanMemFaildMsg, err.Error())
 				return
@@ -185,7 +189,12 @@ func (motor *Engine) checkAndFire(pe *PacketExtractor) {
 				event.Timestamp = &models.MoleTime{
 					Time: metadata.Timestamp,
 				}
-				event.EventType = match.Meta["type"].(string)
+				typ, ok := extractMeta(match.Metas, "type").(string)
+				if !ok {
+					event.EventType = "unkown"
+				} else {
+					event.EventType = typ
+				}
 				event.InIface = pe.GetIfaceName()
 				event.Proto = meta[nodes.Proto.String()].GetValue()
 				event.SrcIP = meta[nodes.SrcNet.String()].GetValue()
@@ -196,7 +205,7 @@ func (motor *Engine) checkAndFire(pe *PacketExtractor) {
 				event.Alert = models.AlertEvent{
 					Name: match.Rule,
 					Tags: match.Tags,
-					Meta: match.Meta,
+					Meta: toMoleMetaMap(match.Metas),
 				}
 
 				var matchArr models.MatchArray
@@ -215,3 +224,7 @@ func (motor *Engine) checkAndFire(pe *PacketExtractor) {
 		}
 	}
 }
+
+func (motor *Engine) ruleMatching(m []yara.MatchRule, err error) {
+
+}

pkg/engine/engine.go

@@ -13,6 +13,11 @@
 // limitations under the License.
 package engine
 
+import (
+	"github.com/hillu/go-yara/v4"
+	"github.com/mole-ids/mole/pkg/logger/models"
+)
+
 // inProtos checks `pkgProto` exists in  `protos`
 func inProtos(proto string, protos []string) bool {
 	for _, p := range protos {
@@ -22,3 +27,22 @@ func inProtos(proto string, protos []string) bool {
 	}
 	return false
 }
+
+func extractMeta(metas []yara.Meta, key string) interface{} {
+	for _, meta := range metas {
+		if meta.Identifier == key {
+			return meta.Value
+		}
+	}
+	return nil
+}
+
+func toMoleMetaMap(metas []yara.Meta) models.MetaMap {
+	var obj models.MetaMap
+	obj = make(models.MetaMap)
+
+	for _, meta := range metas {
+		obj[meta.Identifier] = meta.Value
+	}
+	return obj
+}

pkg/engine/helper.go

@@ -19,7 +19,7 @@ import (
 	"regexp"
 	"strings"
 
-	"github.com/hillu/go-yara"
+	"github.com/hillu/go-yara/v4"
 	"github.com/mole-ids/mole/internal/nodes"
 	"github.com/mole-ids/mole/internal/types"
 	"github.com/mole-ids/mole/internal/utils"
@@ -29,7 +29,7 @@ import (
 // GetRuleMetaInfo returns the rule metadata
 func GetRuleMetaInfo(rule yara.Rule) (metarule types.MetaRule, err error) {
 	metarule = make(types.MetaRule)
-	for _, meta := range rule.MetaList() {
+	for _, meta := range rule.Metas() {
 		if utils.InStrings(meta.Identifier, nodes.Keywords) {
 			// This will never generate an error becauses meta.Identifieris double
 			// checked in the previous conditional

pkg/rules/helpers.go

@@ -21,7 +21,7 @@ import (
 	"strings"
 	"testing"
 
-	"github.com/hillu/go-yara"
+	"github.com/hillu/go-yara/v4"
 	"github.com/mole-ids/mole/internal/nodes"
 	"github.com/mole-ids/mole/internal/types"
 	"github.com/spf13/viper"

pkg/rules/helpers_test.go

@@ -0,0 +1,15 @@
+rule Test1 : Foo Bar {
+    meta:
+        author = "Mole-IDS"
+        type = "alert"
+        uuid = "<not used>"
+        proto = "tcp"
+        src = "any"
+        sport = "80"
+        dst = "any"
+        dport = "any"
+    strings:
+        $method = "GET"
+    condition:
+        $method at 0
+}