Merge pull request #44 from HubbleStack/develop

Merge to master (prep for v2016.10.4)

Author: basepi

FORMULA

@@ -1,7 +1,7 @@
 name: hubblestack_quasar
 os: RedHat, CentOS, Debian, Ubuntu
 os_family: RedHat, Debian
-version: 2016.9.0
+version: 2016.10.4
 release: 1
 summary: HubbleStack Quasar
 description: HubbleStack Quasar

README.rst

@@ -48,8 +48,8 @@ repo for updates and bugfixes!)
 
 .. code-block:: shell
 
-    wget https://spm.hubblestack.io/quasar/hubblestack_quasar-2016.10.3-1.spm
-    spm local install hubblestack_quasar-2016.10.3-1.spm
+    wget https://spm.hubblestack.io/quasar/hubblestack_quasar-2016.10.4-1.spm
+    spm local install hubblestack_quasar-2016.10.4-1.spm
 
 You should now be able to sync the new modules to your minion(s) using the
 ``sync_returners`` Salt utility:
@@ -106,6 +106,25 @@ Target the ``hubblestack_quasar.sls`` extension and target it to selected minion
 
 Once these modules are synced you'll be ready to begin reporting data and events.
 
+Installation (GitFS)
+--------------------
+
+This installation method subscribes directly to our GitHub repository, pinning
+to a tag or branch. This method requires no package installation or manual
+checkouts.
+
+Requirements: GitFS support on your Salt Master.
+
+**/etc/salt/master.d/hubblestack-quasar.conf**
+
+.. code-block:: diff
+
+    gitfs_remotes:
+      - https://github.com/hubblestack/quasar:
+        - base: v2016.10.4
+
+.. tip:: Remember to restart the Salt Master after applying this change.
+
 .. _quasar_usage:
 
 Usage

_returners/slack_pulsar_returner.py

@@ -69,7 +69,7 @@
 # Import Salt Libs
 import salt.returners
 
-__version__ = 'v2016.10.3'
+__version__ = 'v2016.10.4'
 
 log = logging.getLogger(__name__)
 

_returners/splunk_nebula_return.py

@@ -3,7 +3,7 @@
 HubbleStack Nebula-to-Splunk returner
 
 :maintainer: HubbleStack
-:maturity: 2016.7.0
+:maturity: 2016.10.4
 :platform: All
 :requires: SaltStack
 
@@ -20,6 +20,26 @@
             indexer: <hostname/IP of Splunk indexer>
             sourcetype: <Destination sourcetype for data>
             index: <Destination index for data>
+
+You can also add an `custom_fields` argument which is a list of keys to add to events
+with using the results of config.get(<custom_field>). These new keys will be prefixed
+with 'custom_' to prevent conflicts. The values of these keys should be
+strings or lists (will be sent as CSV string), do not choose grains or pillar values with complex values or they will
+be skipped:
+
+.. code-block:: yaml
+
+    hubblestack:
+      nebula:
+        returner:
+          splunk:
+            token: <splunk_http_forwarder_token>
+            indexer: <hostname/IP of Splunk indexer>
+            sourcetype: <Destination sourcetype for data>
+            index: <Destination index for data>
+            custom_fields:
+              - site
+              - product_group
 '''
 import socket
 
@@ -30,7 +50,7 @@
 
 import logging
 
-__version__ = 'v2016.10.3'
+__version__ = 'v2016.10.4'
 
 _max_content_bytes = 100000
 http_event_collector_SSL_verify = False
@@ -51,6 +71,7 @@ def returner(ret):
     hec_ssl = opts['http_event_server_ssl']
     proxy = opts['proxy']
     timeout = opts['timeout']
+    custom_fields = opts['custom_fields']
     # Set up the collector
     hec = http_event_collector(http_event_collector_key, http_event_collector_host, http_event_server_ssl=hec_ssl, proxy=proxy, timeout=timeout)
 
@@ -80,6 +101,16 @@ def returner(ret):
                     event.update({'minion_id': minion_id})
                     event.update({'dest_host': fqdn})
                     event.update({'dest_ip': fqdn_ip4})
+
+                    for custom_field in custom_fields:
+                        custom_field_name = 'custom_' + custom_field
+                        custom_field_value = __salt__['config.get'](custom_field, '')
+                        if isinstance(custom_field_value, str):
+                            event.update({custom_field_name: custom_field_value})
+                        elif isinstance(custom_field_value, list):
+                            custom_field_value = ','.join(custom_field_value)
+                            event.update({custom_field_name: custom_field_value})
+
                     payload.update({'host': fqdn})
                     payload.update({'index': opts['index']})
                     payload.update({'sourcetype': opts['sourcetype']})
@@ -96,9 +127,10 @@ def _get_options():
         indexer = __salt__['config.get']('hubblestack:nebula:returner:splunk:indexer')
         sourcetype = __salt__['config.get']('hubblestack:nebula:returner:splunk:sourcetype')
         index = __salt__['config.get']('hubblestack:nebula:returner:splunk:index')
+        custom_fields = __salt__['config.get']('hubblestack:nebula:returner:splunk:custom_fields', [])
     except:
         return None
-    splunk_opts = {'token': token, 'indexer': indexer, 'sourcetype': sourcetype, 'index': index}
+    splunk_opts = {'token': token, 'indexer': indexer, 'sourcetype': sourcetype, 'index': index, 'custom_fields': custom_fields}
 
     hec_ssl = __salt__['config.get']('hubblestack:nebula:returner:splunk:hec_ssl', True)
     splunk_opts['http_event_server_ssl'] = hec_ssl

_returners/splunk_nova_return.py

@@ -3,7 +3,7 @@
 HubbleStack Nova-to-Splunk returner
 
 :maintainer: HubbleStack
-:maturity: 2016.7.0
+:maturity: 2016.10.4
 :platform: All
 :requires: SaltStack
 
@@ -20,6 +20,26 @@
             indexer: <hostname/IP of Splunk indexer>
             sourcetype: <Destination sourcetype for data>
             index: <Destination index for data>
+
+You can also add an `custom_fields` argument which is a list of keys to add to events
+with using the results of config.get(<custom_field>). These new keys will be prefixed
+with 'custom_' to prevent conflicts. The values of these keys should be
+strings, do not choose grains or pillar values with complex values or they will
+be skipped:
+
+.. code-block:: yaml
+
+    hubblestack:
+      nova:
+        returner:
+          splunk:
+            token: <splunk_http_forwarder_token>
+            indexer: <hostname/IP of Splunk indexer>
+            sourcetype: <Destination sourcetype for data>
+            index: <Destination index for data>
+            custom_fields:
+              - site
+              - product_group
 '''
 import socket
 
@@ -30,7 +50,7 @@
 
 import logging
 
-__version__ = 'v2016.10.3'
+__version__ = 'v2016.10.4'
 
 _max_content_bytes = 100000
 http_event_collector_SSL_verify = False
@@ -50,6 +70,7 @@ def returner(ret):
     hec_ssl = opts['http_event_server_ssl']
     proxy = opts['proxy']
     timeout = opts['timeout']
+    custom_fields = opts['custom_fields']
     # Set up the collector
     hec = http_event_collector(http_event_collector_key, http_event_collector_host, http_event_server_ssl=hec_ssl, proxy=proxy, timeout=timeout)
     # st = 'salt:hubble:nova'
@@ -85,6 +106,16 @@ def returner(ret):
         event.update({'minion_id': minion_id})
         event.update({'dest_host': fqdn})
         event.update({'dest_ip': fqdn_ip4})
+
+        for custom_field in custom_fields:
+            custom_field_name = 'custom_' + custom_field
+            custom_field_value = __salt__['config.get'](custom_field, '')
+            if isinstance(custom_field_value, str):
+                event.update({custom_field_name: custom_field_value})
+            elif isinstance(custom_field_value, list):
+                custom_field_value = ','.join(custom_field_value)
+                event.update({custom_field_name: custom_field_value})
+
         payload.update({'host': fqdn})
         payload.update({'index': opts['index']})
         payload.update({'sourcetype': opts['sourcetype']})
@@ -95,18 +126,30 @@ def returner(ret):
         check_id = suc.keys()[0]
         payload = {}
         event = {}
-        event.update({'minion_id': minion_id})
         event.update({'check_result': 'Success'})
         event.update({'check_id': check_id})
         event.update({'job_id': jid})
-        event.update({'master': master})
         if not isinstance(suc[check_id], dict):
             event.update({'description': suc[check_id]})
         elif 'description' in suc[check_id]:
             for key, value in suc[check_id].iteritems():
                 if key not in ['tag']:
                     event[key] = value
-        payload.update({'host': minion_id})
+        event.update({'master': master})
+        event.update({'minion_id': minion_id})
+        event.update({'dest_host': fqdn})
+        event.update({'dest_ip': fqdn_ip4})
+
+        for custom_field in custom_fields:
+            custom_field_name = 'custom_' + custom_field
+            custom_field_value = __salt__['config.get'](custom_field, '')
+            if isinstance(custom_field_value, str):
+                event.update({custom_field_name: custom_field_value})
+            elif isinstance(custom_field_value, list):
+                custom_field_value = ','.join(custom_field_value)
+                event.update({custom_field_name: custom_field_value})
+
+        payload.update({'host': fqdn})
         payload.update({'sourcetype': opts['sourcetype']})
         payload.update({'index': opts['index']})
         payload.update({'event': event})
@@ -115,11 +158,23 @@ def returner(ret):
     if data.get('Compliance', None):
         payload = {}
         event = {}
-        event.update({'minion_id': minion_id})
         event.update({'job_id': jid})
-        event.update({'master': master})
         event.update({'compliance_percentage': data['Compliance']})
-        payload.update({'host': minion_id})
+        event.update({'master': master})
+        event.update({'minion_id': minion_id})
+        event.update({'dest_host': fqdn})
+        event.update({'dest_ip': fqdn_ip4})
+
+        for custom_field in custom_fields:
+            custom_field_name = 'custom_' + custom_field
+            custom_field_value = __salt__['config.get'](custom_field, '')
+            if isinstance(custom_field_value, str):
+                event.update({custom_field_name: custom_field_value})
+            elif isinstance(custom_field_value, list):
+                custom_field_value = ','.join(custom_field_value)
+                event.update({custom_field_name: custom_field_value})
+
+        payload.update({'host': fqdn})
         payload.update({'sourcetype': opts['sourcetype']})
         payload.update({'index': opts['index']})
         payload.update({'event': event})
@@ -154,9 +209,10 @@ def _get_options():
         indexer = __salt__['config.get']('hubblestack:nova:returner:splunk:indexer')
         sourcetype = __salt__['config.get']('hubblestack:nova:returner:splunk:sourcetype')
         index = __salt__['config.get']('hubblestack:nova:returner:splunk:index')
+        custom_fields = __salt__['config.get']('hubblestack:nebula:returner:splunk:custom_fields', [])
     except:
         return None
-    splunk_opts = {'token': token, 'indexer': indexer, 'sourcetype': sourcetype, 'index': index}
+    splunk_opts = {'token': token, 'indexer': indexer, 'sourcetype': sourcetype, 'index': index, 'custom_fields': custom_fields}
 
     hec_ssl = __salt__['config.get']('hubblestack:nova:returner:splunk:hec_ssl', True)
     splunk_opts['http_event_server_ssl'] = hec_ssl

_returners/splunk_pulsar_return.py

@@ -3,7 +3,7 @@
 HubbleStack Pulsar-to-Splunk returner
 
 :maintainer: HubbleStack
-:maturity: 2016.7.0
+:maturity: 2016.10.4
 :platform: All
 :requires: SaltStack
 
@@ -20,6 +20,26 @@
             indexer: <hostname/IP of Splunk indexer>
             sourcetype: <Destination sourcetype for data>
             index: <Destination index for data>
+
+You can also add an `custom_fields` argument which is a list of keys to add to events
+with using the results of config.get(<custom_field>). These new keys will be prefixed
+with 'custom_' to prevent conflicts. The values of these keys should be
+strings, do not choose grains or pillar values with complex values or they will
+be skipped:
+
+.. code-block:: yaml
+
+    hubblestack:
+      pulsar:
+        returner:
+          splunk:
+            token: <splunk_http_forwarder_token>
+            indexer: <hostname/IP of Splunk indexer>
+            sourcetype: <Destination sourcetype for data>
+            index: <Destination index for data>
+            custom_fields:
+              - site
+              - product_group
 '''
 
 import socket
@@ -33,7 +53,7 @@
 
 import logging
 
-__version__ = 'v2016.10.3'
+__version__ = 'v2016.10.4'
 
 _max_content_bytes = 100000
 http_event_collector_SSL_verify = False
@@ -53,6 +73,7 @@ def returner(ret):
     hec_ssl = opts['http_event_server_ssl']
     proxy = opts['proxy']
     timeout = opts['timeout']
+    custom_fields = opts['custom_fields']
     # Set up the collector
     hec = http_event_collector(http_event_collector_key, http_event_collector_host, http_event_server_ssl=hec_ssl, proxy=proxy, timeout=timeout)
     # Check whether or not data is batched:
@@ -164,6 +185,16 @@ def returner(ret):
         event.update({'minion_id': minion_id})
         event.update({'dest_host': fqdn})
         event.update({'dest_ip': fqdn_ip4})
+
+        for custom_field in custom_fields:
+            custom_field_name = 'custom_' + custom_field
+            custom_field_value = __salt__['config.get'](custom_field, '')
+            if isinstance(custom_field_value, str):
+                event.update({custom_field_name: custom_field_value})
+            elif isinstance(custom_field_value, list):
+                custom_field_value = ','.join(custom_field_value)
+                event.update({custom_field_name: custom_field_value})
+
         payload.update({'host': fqdn})
         payload.update({'index': opts['index']})
         payload.update({'sourcetype': opts['sourcetype']})
@@ -188,9 +219,10 @@ def _get_options():
         indexer = __salt__['config.get']('hubblestack:pulsar:returner:splunk:indexer')
         sourcetype = __salt__['config.get']('hubblestack:pulsar:returner:splunk:sourcetype')
         index = __salt__['config.get']('hubblestack:pulsar:returner:splunk:index')
+        custom_fields = __salt__['config.get']('hubblestack:nebula:returner:splunk:custom_fields', [])
     except:
         return None
-    splunk_opts = {'token': token, 'indexer': indexer, 'sourcetype': sourcetype, 'index': index}
+    splunk_opts = {'token': token, 'indexer': indexer, 'sourcetype': sourcetype, 'index': index, 'custom_fields': custom_fields}
 
     hec_ssl = __salt__['config.get']('hubblestack:pulsar:returner:splunk:hec_ssl', True)
     splunk_opts['http_event_server_ssl'] = hec_ssl