Merge branch 'devel'

Author: Petr Velan

Makefile.am

@@ -1,6 +1,6 @@
 ACLOCAL_AMFLAGS = -I m4 
 
-EXTRA_DIST = README.md LICENSE
+EXTRA_DIST = README.md LICENSE docker
 
 SUBDIRS = base \
 	tools/fbitconvert \

README.md

@@ -18,9 +18,10 @@
     *  [fbitmerge](#fbitmerge)
 	*  [profilesdaemon](#profilesdaemon)
 6.  [Howto build](#build)
-7.  [RPM](#rpm)
-8.  [FastBit](#fastbit)
-9.  [Contact us](#contact)
+7.  [Docker](#docker)
+8.  [RPM](#rpm)
+9.  [FastBit](#fastbit)
+10.  [Contact us](#contact)
     *  [Reporting bugs](#bug)
     *  [Forum](#mailing)
 
@@ -79,7 +80,7 @@ More info in it's [README](tools/fbitdump/)
 
 ###<a name="fbitexpire"></a> fbitexpire
 
-Daemon for removal old data.
+Daemon for removing old data.
 
 More info in it's [README](tools/fbitexpire/)
 
@@ -118,6 +119,14 @@ Or you can build each part (collector, tool(s), extarnal plugin(s)) separately.
 The projects that depend on ipfixcol headers check the reltive path to base/header directory to use headers. 
 When project is separated from the structure, it needs to have the headers installed (ipfixcol-devel package).
 
+##<a name="docker"></a> Docker
+
+IPFIXcol can be used with Docker. See [Docker howto](docker).
+
+##<a name="ansible"></a> Ansible
+
+IPFIXcol can also be installed using Ansible orchestration. See [Ansible howto](ansible).
+
 ##<a name="rpm"></a> RPM
 Each part of framework supports building rpm packages by running
 

ansible/README.md

@@ -0,0 +1,38 @@
+# <a name="top"></a>Installing IPFIXcol using Ansible
+
+If you are familiar with Ansible, you can use it to easily install IPFIXcol on any target machine. Provided playbook allows you to separately install:
+* dependencies for IPFIXcol
+* IPFIXcol base
+* any IPFIXcol plugin (requires base first)
+* IPFIXcol tools (e.g. fbitdump)
+
+The Ansible orchestration was tested on following systems:
+* Debian Jessie (8.5)
+* Ubuntu Xenial (16.04 LTS)
+* CentOS 7 (7.2.1511)
+* Fedora 24
+
+If you want to orchestrate latest Fedora systems, make sure your Ansible version contains the `dnf` module (versions >= 1.9.4). Tested version is 2.1.0.
+
+## <a name="howto"></a>Howto use IPFIXcol playbook
+
+Installing IPFIXcol with Ansible is fairly simple. First, put target machines into the hosts file. The default target is localhost.
+```
+[ipfixcol-hosts]
+localhost
+```
+To install the entire collector including all (exceptions apply) plugins and tools, use
+```
+ansible-playbook -i hosts ipfixcol.yml
+```
+To see what this command would do, use `--check` switch with `ansible-playbook`
+
+If you want only dependencies for basic IPFIXcol without plugins, use
+```
+ansible-playbook -i hosts ipfixcol.yml --tags dependencies
+```
+There is a tag for each plugin and its dependency. LibFastbit library can be installed separately as well.
+
+## <a name="libfastbit"></a>LibFastbit compilation
+
+The LibFastbit is a fairly large beast and takes some time to compile. Therefore, we usually install it using binaries, which are pre-compiled for all supported systems (see above). However, if you are installing on unsupported an distribution or you just want to build from sources, you can force this by using `--extra-vars "build_fastbit_compile=true"`

ansible/hosts

@@ -0,0 +1,2 @@
+[ipfixcol-hosts]
+localhost

ansible/ipfixcol.yml

@@ -0,0 +1,3 @@
+- hosts: ipfixcol-hosts
+  roles:
+    - ipfixcol

ansible/roles/ipfixcol/tasks/base.yml

@@ -0,0 +1,13 @@
+# IPFIXcol build
+- name: Clone IPFIXcol git
+  git: repo={{ build.git }} dest={{ build.dir }} version={{ build.branch }}
+
+- name: Build IPFIXcol base
+  command: >
+    {{ item }}
+    chdir={{ build.dir }}/base/
+    creates=/tmp/ipfixcol-base.installed
+  with_items:
+    - "{{ build.cmds }}"
+    - ldconfig
+    - "touch /tmp/ipfixcol-base.installed"

ansible/roles/ipfixcol/tasks/dependencies.yml

@@ -0,0 +1,55 @@
+# Dependencies for IPFIXcol base
+- name: Check for yum packaging system
+  set_fact:
+    pkg_type: yum
+  when: ansible_pkg_mgr == 'yum' or ansible_pkg_mgr == 'dnf'
+  tags: always
+- name: Check for apt packaging system
+  set_fact:
+    pkg_type: apt
+  when: ansible_pkg_mgr == 'apt'
+  tags: always
+
+- name: Include dependency package variables
+  include_vars: "packages.{{ pkg_type }}.yml"
+  tags: always
+  
+- name: Install dnf support for Fedora
+  command: dnf -y install python2-dnf
+  when: ansible_pkg_mgr == 'dnf'
+  tags: dependencies
+
+- name: Install dependencies for IPFIXcol
+  action: "{{ ansible_pkg_mgr }} name={{ item }} state=latest"
+  with_items: "{{ packages.base }}"
+  tags: dependencies
+  
+- name: Download FastBit compiled library
+  include: libfastbit.yml
+  tags: 
+   - fastbit_lib
+   - fastbit
+   - fbitmerge
+   - fbitdump
+
+- name: postgres dependencies
+  action: "{{ ansible_pkg_mgr }} name={{ item }} state=latest"
+  with_items: "{{ packages.postgres }}"
+  tags: postgres
+
+- name: GeoIP dependencies
+  action: "{{ ansible_pkg_mgr }} name={{ item }} state=latest"
+  with_items: "{{ packages.geoip }}"
+  tags: geoip
+  
+- name: stats dependencies
+  action: "{{ ansible_pkg_mgr }} name={{ item }} state=latest"
+  with_items: "{{ packages.stats }}"
+  tags:
+    - profile_stats
+    - stats
+    
+- name: uid dependencies
+  action: "{{ ansible_pkg_mgr }} name={{ item }} state=latest"
+  with_items: "{{ packages.uid }}"
+  tags: uid

ansible/roles/ipfixcol/tasks/input-plugins.yml

@@ -0,0 +1,19 @@
+# Build input plugins
+
+# nfdump
+- name: Build nfdump input plugin
+  command: >
+    {{ item }}
+    chdir={{ build.dir }}/plugins/input/nfdump
+    creates=/usr/local/share/ipfixcol/plugins/ipfixcol-nfdump-input.so
+  with_items: "{{ build.cmds }}"
+  tags: nfdump-input
+
+# udp_cpg - requires libcpg not available fedora and centos repos
+#- name: Build UDP CPG input plugin
+#  command: >
+#    {{ item }}
+#    chdir={{ build.dir }}/plugins/input/udp_cpg
+#    creates=/usr/local/share/ipfixcol/plugins/ipfixcol-udp_cpg-input.so
+#  with_items: "{{ build.cmds }}"
+#  tags: udp_cpg

ansible/roles/ipfixcol/tasks/intermediate-plugins.yml

@@ -0,0 +1,47 @@
+# Intermediate plugins
+
+# geoip
+- name: Build GeoIP intermediate plugin
+  command: >
+    {{ item }}
+    chdir={{ build.dir }}/plugins/intermediate/geoip
+    creates=/usr/local/share/ipfixcol/plugins/ipfixcol-geoip-inter.so
+  with_items: "{{ build.cmds }}"
+  tags: geoip
+
+# profiler
+- name: Build profiler intermediate plugin
+  command: >
+    {{ item }}
+    chdir={{ build.dir }}/plugins/intermediate/profiler
+    creates=/usr/local/share/ipfixcol/plugins/ipfixcol-profiler-inter.so
+  with_items: "{{ build.cmds }}"
+  tags: profiler
+
+
+# profile_stats
+- name: Build profile_stats intermediate plugin
+  command: >
+    {{ item }}
+    chdir={{ build.dir }}/plugins/intermediate/profile_stats
+    creates=/usr/local/share/ipfixcol/plugins/ipfixcol-profilestats-inter.so
+  with_items: "{{ build.cmds }}"
+  tags: profile_stats
+
+# stats
+- name: Build stats intermediate plugin
+  command: >
+    {{ item }}
+    chdir={{ build.dir }}/plugins/intermediate/stats
+    creates=/usr/local/share/ipfixcol/plugins/ipfixcol-stats-inter.so
+  with_items: "{{ build.cmds }}"
+  tags: stats
+
+# uid
+- name: Build uid intermediate plugin
+  command: >
+    {{ item }}
+    chdir={{ build.dir }}/plugins/intermediate/uid
+    creates=/usr/local/share/ipfixcol/plugins/ipfixcol-uid-inter.so
+  with_items: "{{ build.cmds }}"
+  tags: uid

ansible/roles/ipfixcol/tasks/libfastbit.yml

@@ -0,0 +1,40 @@
+# Install libfastbit by either downloading precompiled
+# binaries or combiling from git
+
+# Install using binaries
+- name: Download FastBit compiled library
+  get_url: >
+    url={{ build.fastbit.url }}/libfastbit-{{ ansible_distribution }}-{{ ansible_distribution_major_version }}.tar.gz
+    dest=/tmp/{{ build.fastbit.archive }}
+  when: not (build_fastbit_compile | bool)
+
+- name: Install FastBit library
+  command: "{{ item }} chdir=/tmp/ creates=/tmp/libfastbit.installed"
+  with_items:
+    - "tar -xzpf {{ build.fastbit.archive }} -C /"
+    # move pkgconfig file to path common to all supported distributions
+    - "mv /usr/local/lib/pkgconfig/fastbit.pc /usr/share/pkgconfig/fastbit.pc"
+    - "ldconfig"
+    - "touch /tmp/libfastbit.installed"
+  when: not (build_fastbit_compile | bool)
+
+# Install from sources (https://github.com/CESNET/libfastbit)
+- name: Clone libfastbit repository
+  git: repo={{ build.fastbit.git }} dest={{ build.fastbit.dir }}
+  when: build_fastbit_compile | bool
+  ignore_errors: yes 
+
+- name: Build libfastbit
+  command: >
+    {{ item }}
+    chdir={{ build.fastbit.dir }}
+    creates=/tmp/libfastbit.installed
+  with_items:
+    - "autoreconf -i"
+    - "./configure --disable-static"
+    - "make"
+    - "make install"
+    - "mv /usr/local/lib/pkgconfig/fastbit.pc /usr/share/pkgconfig/fastbit.pc"
+    - "ldconfig"
+    - "touch /tmp/libfastbit.installed"
+  when: build_fastbit_compile | bool