Merge pull request #1 from arruko/feature/jhg/ANS-001

ANS-001

defaults/main.yml

@@ -0,0 +1,13 @@
+---
+logstash_major_ver: 6.x
+logstash_repo_key: https://artifacts.elastic.co/GPG-KEY-elasticsearch
+logstash_deb_repo: "deb https://artifacts.elastic.co/packages/{{ logstash_major_ver }}/apt stable main"
+logstash_listen_port_beats: 5044
+logstash_local_syslog_path: /var/log/syslog
+logstash_monitor_local_syslog: true
+logstash_enabled_on_boot: yes
+logstash_min_memory_required: 2048
+logstash_elasticsearch_hosts: "{{ groups['elasticsearch'] | map('extract', hostvars, ['ansible_host']) | list }}"
+logstash_elasticsearch_inventory_group_name: elasticsearch
+logstash_install_plugins:
+  - logstash-input-beats

files/filters/nginx.conf

@@ -0,0 +1,7 @@
+filter {
+  if [type] == "nginx" {
+    grok {
+      match => { "message" => "%{COMBINEDAPACHELOG}" }
+    }
+  }
+}

files/filters/syslog.conf

@@ -0,0 +1,16 @@
+filter {
+  if [type] == "syslog" {
+    if [message] =~ /last message repeated [0-9]+ times/ {
+      drop { }
+    }
+    grok {
+      match => { "message" => "%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:syslog_hostname} %{DATA:syslog_program}(?:\[%{POSINT:syslog_pid}\])?: %{GREEDYDATA:syslog_message}" }
+      add_field => [ "received_at", "%{@timestamp}" ]
+      add_field => [ "received_from", "%{host}" ]
+    }
+    syslog_pri { }
+    date {
+      match => [ "syslog_timestamp", "MMM  d HH:mm:ss", "MMM dd HH:mm:ss" ]
+    }
+  }
+}

handlers/main.yml

@@ -0,0 +1,3 @@
+---
+- name: restart logstash
+  service: name=logstash state=restarted

tasks/config.yml

@@ -0,0 +1,41 @@
+---
+- name: config | create Logstash configuration files
+  template:
+    src: "{{ item }}.j2"
+    dest: "/etc/logstash/conf.d/{{ item }}"
+    owner: root
+    group: root
+    mode: 0644
+  with_items:
+    - 01-beats-input.conf
+    - 03-elasticsearch-output.conf
+  notify: restart logstash
+
+- name: config | create Logstash filters
+  copy:
+    src: "filters/{{ item }}"
+    dest: "/etc/logstash/conf.d/{{ item }}"
+    owner: root
+    group: root
+    mode: 0644
+  with_items:
+    - syslog.conf
+    - nginx.conf
+  notify: restart logstash
+
+- name: config | create Logstash configuration file for local syslog
+  template:
+    src: 02-local-syslog-input.conf.j2
+    dest: /etc/logstash/conf.d/02-local-syslog-input.conf
+    owner: root
+    group: root
+    mode: 0644
+  when: logstash_monitor_local_syslog
+  notify: restart logstash
+
+- name: config | ensure configuration for local syslog is absent if disabled
+  file:
+    path: /etc/logstash/conf.d/02-local-syslog-input.conf
+    state: absent
+  when: not logstash_monitor_local_syslog
+  notify: restart logstash

tasks/main.yml

@@ -0,0 +1,26 @@
+---
+- name: Test distribution
+  assert:
+    that: >
+      ansible_os_family == 'RedHat' or
+      ansible_os_family == 'Debian'
+
+- name: Include OS Specific setup tasks
+  include: setup-{{ ansible_os_family }}.yml
+
+- name: Checking To Ensure Node Meets Minimum Specs
+  pause:
+    prompt: >
+         "Node Does Not Meet Minimum Recommend Memory Requirements of
+         {{ logstash_min_memory_required }}MB"
+    seconds: 10
+  when: ansible_memtotal_mb < logstash_min_memory_required
+
+- include: config.yml
+- include: plugins.yml
+
+- name: Ensure Logstash is started and enabled on boot.
+  service:
+    name: logstash
+    state: started
+    enabled: "{{ logstash_enabled_on_boot }}"

tasks/plugins.yml

@@ -0,0 +1,23 @@
+---
+- name: plugins | Setting Logstash plugins facts
+  set_fact:
+    logstash_plugin_bin: '/usr/share/logstash/bin/logstash-plugin'
+    logstash_bin: '/usr/share/logstash/bin/logstash'
+  when: logstash_major_ver == '5.x'
+
+- name: plugins | collecting installed plugins
+  command: "{{ logstash_plugin_bin }} list"
+  become: true
+  register: "logstash_plugins_installed"
+  changed_when: false
+
+- name: plugins | Currently Installed Plugins
+  debug: var=logstash_plugins_installed.stdout_lines
+  changed_when: false
+
+- name: plugins | installing logstash plugins
+  shell: "{{ logstash_plugin_cmd_vars }} {{ logstash_plugin_bin }} install {{ item }}"
+  become: true
+  notify: "restart logstash"
+  with_items: '{{ logstash_install_plugins }}'
+  when: item not in logstash_plugins_installed.stdout

tasks/setup-Debian.yml

@@ -0,0 +1,34 @@
+---
+- name: Debian | add apt-transport-https
+  apt:
+    name: apt-transport-https
+    state: present
+
+- name: Debian | add Elasticsearch apt key.
+  apt_key:
+    url: "{{ logstash_repo_key }}"
+    state: present
+
+- name: Debian | add Logstash repository.
+  apt_repository:
+    repo: "{{ logstash_deb_repo }}"
+    state: present
+
+- name: Debian | check if Logstash is already installed.
+  stat: path=/etc/init.d/logstash
+  register: logstash_installed
+
+- name: Debian | update apt cache if repository just added.
+  apt: update_cache=yes
+  when: logstash_installed.stat.exists == false
+
+- name: Debian | install Logstash.
+  apt: pkg=logstash state=present
+
+- name: Debian | add Logstash user to adm group (Debian).
+  user:
+    name: logstash
+    group: logstash
+    groups: adm
+  when: ansible_os_family == "Debian"
+  notify: restart logstash

tasks/setup-RedHat.yml

@@ -0,0 +1,14 @@
+---
+- name: RHEL | add Elasticsearch GPG key.
+  rpm_key:
+    key: http://packages.elasticsearch.org/GPG-KEY-elasticsearch
+    state: present
+
+- name: RHEL | add Logstash repository.
+  copy:
+    src: logstash.repo
+    dest: /etc/yum.repos.d/logstash.repo
+    mode: 0644
+
+- name: RHEL | install Logstash.
+  yum: pkg=logstash state=installed

templates/01-beats-input.conf.j2

@@ -0,0 +1,5 @@
+input {
+  beats {
+    port => {{ logstash_listen_port_beats }}
+  }
+}

templates/02-local-syslog-input.conf.j2

@@ -0,0 +1,5 @@
+input {
+  file {
+    path => "{{ logstash_local_syslog_path }}"
+  }
+}

templates/03-elasticsearch-output.conf.j2

@@ -0,0 +1,7 @@
+output {
+  elasticsearch {
+    hosts => {{ logstash_elasticsearch_hosts | to_json }}
+    index => "%{[@metadata][beat]}-%{+YYYY.MM.dd}"
+    document_type => "%{[@metadata][type]}"
+  }
+}