Commit 92f99efa authored by Roman Kiš's avatar Roman Kiš
Browse files

k8s & ansible playbook

without persist volume and ingress
parent f545df43
# Elastic Stack
---
# misc variables
project: nghis
docker_registry: nexus.assecosk.local:8080
# Namespace into which the application should be deployed
namespace: elk
# Domain name for the application
dns_zone: dev.prosoftke.sk
# External SSL certificates
key_path: /etc/prosoft/ssl/prosoft.key
cert_path: /etc/prosoft/ssl/prosoft.crt
- hosts: localhost
connection: local
roles:
- elasticsearch
- kibana
- logspout
- logstash
####################
# Install elasticsearch
####################
- name: Generate deplyoment elasticsearch
template:
src: elasticsearch.yaml.j2
dest: /tmp/elk/elasticsearch.yaml
mode: 0744
become: false
- name: Deploy Elasticsearch
command: 'kubectl --namespace {{ namespace }} apply -f /tmp/elk/elasticsearch.yaml'
apiVersion: apps/v1
kind: StatefulSet
metadata:
namespace: {{ namespace }}
name: {{ fulname }}
labels:
app: {{ name }}
spec:
serviceName: {{ fullname }}
replicas: 1
selector:
matchLabels:
app: {{ name }}
template:
metadata:
name: {{ name }}
labels:
app: {{ name }}
spec:
containers:
- image: {{ image }}:{{ imageTag }}
name: elasticsearch
env:
- name: LOGSPOUT
value: ignore
ports:
- containerPort: 9200
name: db
protocol: TCP
- containerPort: 9300
name: transport
protocol: TCP
---
apiVersion: v1
kind: Service
metadata:
namespace: {{ namespace }}
name: {{ fullname }}
spec:
type: NodePort
selector:
app: {{ name }}
ports:
- name: http
port: 9200
protocol: TCP
- name: transport
port: 9300
protocol: TCP
name: elasticsearch
fullname: elasticsearch
image: elasticsearch
imageTag: 6.5.4
####################
# Install Kibana
####################
- name: Generate deplyoment Kibana
template:
src: kibana.yaml.j2
dest: /tmp/elk/kibana.yaml
mode: 0744
become: false
- name: Deploy Kibana
command: 'kubectl --namespace {{ namespace }} apply -f /tmp/elk/kibana.yaml'
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
namespace: {{ namespace }}
name: {{ fullname }}
labels:
app: {{ name }}
spec:
replicas: 1
template:
metadata:
name: {{ fullname }}
labels:
app: {{ name }}
spec:
containers:
- image: {{ image }}:{{ imageTag }}
name: {{ fullname }}
env:
- name: LOGSPOUT
value: ignore
ports:
- containerPort: 5601
env:
- name: 'ELASTICSEARCH_URL'
value: 'http://$(ELASTICSEARCH_SERVICE_HOST):$(ELASTICSEARCH_SERVICE_PORT_HTTP)'
---
apiVersion: v1
kind: Service
metadata:
name: {{ fullname }}
spec:
type: NodePort
selector:
app: {{ name }}
ports:
- port: 5601
protocol: TCP
targetPort: 5601
nodePort: 30000
name: kibana
fullname: kibana
image: kibana
imageTag: 6.5.4
####################
# Install Logspout
####################
- name: Generate deplyoment Logspout
template:
src: logspout.yaml.j2
dest: /tmp/elk/logspout.yaml
mode: 0744
become: false
- name: Deploy Logspout
command: 'kubectl --namespace {{ namespace }} apply -f /tmp/elk/logspout.yaml'
apiVersion: v1
kind: ServiceAccount
metadata:
name: logspout
namespace: kube-system
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
namespace: {{ namespace }}
name: {{ fullname }}
labels:
app: {{ name }}
spec:
selector:
matchLabels:
app: {{ name }}
template:
metadata:
labels:
app: {{ name }}
spec:
containers:
- name: {{ fullname }}
image: {{ image }}:{{ imageTag }}
command: ['/bin/logspout']
args:
['syslog://$(LOGSTASH_SERVICE_HOST):$(LOGSTASH_SERVICE_PORT_TCP)']
volumeMounts:
- name: docker
mountPath: /tmp/docker.sock
volumes:
- name: docker
hostPath:
path: /var/run/docker.sock
name: logspout
fullname: logspout
image: gliderlabs/logspout
imageTag: v3.2.6
####################
# Install Logstash
####################
- name: Generate deplyoment logstash
template:
src: logstash.yaml.j2
dest: /tmp/elk/logstash.yaml
mode: 0744
become: false
- name: Create Logstash configmap
command: kubectl create configmap logstash-config --from-file=logstash.conf
- name: Deploy Logstash
command: 'kubectl --namespace {{ namespace }} apply -f /tmp/elk/logstash.yaml'
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
namespace: {{ namespace }}
name: {{ fullname }}
labels:
app: {{ name }}
spec:
replicas: 1
template:
metadata:
name: {{ fullname }}
labels:
app: {{ name }}
spec:
containers:
- image: {{ image }}:{{ imageTag }}
name: {{ fullname }}
env:
- name: LOGSPOUT
value: ignore
command: ['logstash', '-f', 'logstash.conf']
ports:
- containerPort: 5000
protocol: TCP
- containerPort: 5000
protocol: UDP
volumeMounts:
- name: 'logstash-config'
mountPath: '/{{ logstashConfigFile }}'
subPath: '{{ logstashConfigFile }}'
volumes:
- name: 'logstash-config'
configMap:
name: 'logstash-config'
---
apiVersion: v1
kind: Service
metadata:
name: {{ fullname }}
spec:
selector:
app: {{ name }}
ports:
- port: 5000
protocol: TCP
name: tcp
- port: 5000
protocol: UDP
name: udp
name: logstash
fullname: logstash
image: logstash
imageTag: 6.5.4
logstashConfigFile: logstash.conf
......@@ -10,8 +10,8 @@ services:
- '/var/run/docker.sock:/tmp/docker.sock'
logstash:
image: logstash:5
command: 'logstash -f /logstash.conf -e'
image: logstash:6.5.4
command: 'logstash -f /logstash.conf'
links:
- elasticsearch
volumes:
......@@ -20,12 +20,12 @@ services:
- LOGSPOUT=ignore
elasticsearch:
image: elasticsearch:5
image: elasticsearch:6.5.4
environment:
- LOGSPOUT=ignore
kibana:
image: kibana:5
image: kibana:6.5.4
links:
- 'elasticsearch'
ports:
......
input {
syslog {
port => 5000
type => "docker"
}
}
filter {
grok {
match => { "message" => "%{SYSLOG5424PRI}%{NONNEGINT:ver} +(?:%{TIMESTAMP_ISO8601:ts}|-) +(?:%{HOSTNAME:service}|-) +(?:%{NOTSPACE:containerName}|-) +(?:%{NOTSPACE:proc}|-) +(?:%{WORD:msgid}|-) +(?:%{SYSLOG5424SD:sd}|-|) +%{GREEDYDATA:msg}" }
}
syslog_pri { }
date {
match => [ "syslog_timestamp", "MMM d HH:mm:ss", "MMM dd HH:mm:ss" ]
}
mutate {
remove_field => [ "message", "priority", "ts", "severity", "facility", "facility_label", "severity_label", "syslog5424_pri", "proc", "syslog_severity_code", "syslog_facility_code", "syslog_facility", "syslog_severity", "syslog_hostname", "syslog_message", "syslog_timestamp", "ver" ]
}
mutate {
remove_tag => [ "_grokparsefailure_sysloginput" ]
}
mutate {
gsub => [
"service", "[0123456789-]", ""
]
}
if [msg] =~ "^ *{" {
json {
source => "msg"
}
if "_jsonparsefailure" in [tags] {
drop {}
}
mutate {
remove_field => [ "msg" ]
}
}
if ("" in [msg]) {
mutate {
rename => { "msg" => "message" }
}
}
mutate {
remove_field => [ "tags" ]
}
}
output {
elasticsearch {
hosts => "elasticsearch:9200"
}
}
\ No newline at end of file
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
name: elasticsearch
labels:
app: elasticsearch
spec:
replicas: 1
template:
metadata:
name: elasticsearch
labels:
app: elasticsearch
spec:
containers:
- image: elasticsearch:5
name: elasticsearch
env:
- name: LOGSPOUT
value: ignore
ports:
- containerPort: 9200
name: db
protocol: TCP
- containerPort: 9300
name: transport
protocol: TCP
---
apiVersion: v1
kind: Service
metadata:
name: elasticsearch
spec:
type: NodePort
selector:
app: elasticsearch
ports:
- name: http
port: 9200
protocol: TCP
- name: transport
port: 9300
protocol: TCP
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
name: kibana
labels:
app: kibana
spec:
replicas: 1
template:
metadata:
name: kibana
labels:
app: kibana
spec:
containers:
- image: kibana:5
name: kibana
env:
- name: LOGSPOUT
value: ignore
ports:
- containerPort: 5601
env:
- name: 'ELASTICSEARCH_URL'
value: 'http://$(ELASTICSEARCH_SERVICE_HOST):$(ELASTICSEARCH_SERVICE_PORT_HTTP)'
---
apiVersion: v1
kind: Service
metadata:
name: kibana
spec:
type: NodePort
selector:
app: kibana
ports:
- port: 5601
protocol: TCP
targetPort: 5601
nodePort: 30000
#!/bin/sh
kubectl create configmap logstash-config --from-file=logstash.conf
kubectl apply -f elastic.yaml
kubectl apply -f kibana.yaml
kubectl apply -f logstash.yaml
kubectl apply -f logspout.yaml
\ No newline at end of file
apiVersion: v1
kind: ServiceAccount
metadata:
name: logspout
namespace: kube-system
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: logspout
labels:
app: logspout
spec:
selector:
matchLabels:
app: logspout
template:
metadata:
labels:
app: logspout
spec:
containers:
- name: logspout
image: gliderlabs/logspout
command: ['/bin/logspout']
args:
['syslog://$(LOGSTASH_SERVICE_HOST):$(LOGSTASH_SERVICE_PORT_TCP)']
volumeMounts:
- name: docker
mountPath: /tmp/docker.sock
volumes:
- name: docker
hostPath:
path: /var/run/docker.sock
input {
syslog {
port => 5000
type => "docker"
}
}
filter {
grok {
match => { "message" => "%{SYSLOG5424PRI}%{NONNEGINT:ver} +(?:%{TIMESTAMP_ISO8601:ts}|-) +(?:%{HOSTNAME:service}|-) +(?:%{NOTSPACE:containerName}|-) +(?:%{NOTSPACE:proc}|-) +(?:%{WORD:msgid}|-) +(?:%{SYSLOG5424SD:sd}|-|) +%{GREEDYDATA:msg}" }
}
syslog_pri { }
date {
match => [ "syslog_timestamp", "MMM d HH:mm:ss", "MMM dd HH:mm:ss" ]
}
mutate {
remove_field => [ "message", "priority", "ts", "severity", "facility", "facility_label", "severity_label", "syslog5424_pri", "proc", "syslog_severity_code", "syslog_facility_code", "syslog_facility", "syslog_severity", "syslog_hostname", "syslog_message", "syslog_timestamp", "ver" ]
}
mutate {
remove_tag => [ "_grokparsefailure_sysloginput" ]
}
mutate {
gsub => [
"service", "[0123456789-]", ""
]
}
if [msg] =~ "^ *{" {
json {
source => "msg"
}
if "_jsonparsefailure" in [tags] {
drop {}
}
mutate {
remove_field => [ "msg" ]
}
}
if ("" in [msg]) {
mutate {
rename => { "msg" => "message" }
}
}
mutate {
remove_field => [ "tags" ]
}
}
output {
elasticsearch {
hosts => "elasticsearch:9200"
}
}
\ No newline at end of file
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
name: logstash
labels:
app: logstash
spec:
replicas: 1
template:
metadata:
name: logstash
labels:
app: logstash
spec:
containers:
- image: logstash:5
name: logstash
env:
- name: LOGSPOUT
value: ignore
command: ['logstash', '-f', 'logstash.conf']
ports:
- containerPort: 5000
protocol: TCP
- containerPort: 5000
protocol: UDP
volumeMounts:
- name: 'logstash-config'
mountPath: '/logstash.conf'
subPath: 'logstash.conf'
volumes:
- name: 'logstash-config'
configMap:
name: 'logstash-config'
---
apiVersion: v1
kind: Service
metadata:
name: logstash
spec:
selector:
app: logstash
ports:
- port: 5000
protocol: TCP
name: tcp
- port: 5000
protocol: UDP
name: udp
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment