ELK & SENSU
Junyangz
3月 07, 2017
ELK & SENSU
SENSU
Sensu Server and API installation
Install Redis and Configure Redis
sudo apt-get -y install redis-server
sudo update-rc.d redis-server defaults
sudo vim /etc/redis/redis.conf
Change “bind 127.0.0.1” to “bind 0.0.0.0” #VPC USE ONLY!
sudo service redis-server restart
redis-cli ping
return pong and redis is ok.
Install sensu server
wget -q https://sensu.global.ssl.fastly.net/apt/pubkey.gpg -O- |- sudo apt-key add -
echo "deb https://sensu.global.ssl.fastly.net/apt sensu main" | sudo tee /etc/apt/sources.list.d/sensu.list
sudo apt-get update && sudo apt-get install sensu
Config sensu server
sudo vim /etc/sensu/conf.d/api.json
{
"api":{
"host":"0.0.0.0",
"port":4567
}
}
sudo vim /etc/sensu/conf.d/client.json
{
"client":{
"name":"sensu-server",
"address":" local-ip ",
"environment":"sensu",
"subscriptions":[
" linux ",
" dev "
],
"socket":{
"bind":"127.0.0.1",
"port":3030
}
}
}
sudo vim /etc/sensu/conf.d/redis.json
{
"redis":{
"host":" local-ip(redis install in localhost) ",
"port":6379,
"reconnect_on_error":true,
"auto_reconnect":true
}
}
sudo vim /etc/sensu/conf.d/transport.json
{
"transport":{
"name":"redis",
"reconnect_on_error":true
}
}
Enable the Sensu services to start on boot and Start Sensu
sudo update-rc.d sensu-server defaults
sudo update-rc.d sensu-api defaults
sudo update-rc.d sensu-client defaults
sudo service sensu-server start && sudo service sensu-api start &&- sudo service sensu-client start
Installation on the Client
Install sensu client
wget -q https://sensu.global.ssl.fastly.net/apt/pubkey.gpg -O- |- sudo apt-key add -
echo "deb https://sensu.global.ssl.fastly.net/apt sensu main" | sudo- tee /etc/apt/sources.list.d/sensu.list
sudo apt-get update && sudo apt-get install sensu uchiwa
Configure sensu client and uchiwa
sudo vim /etc/sensu/conf.d/client.json
{
"client":{
"name":" hostname ",
"address":" local-ip ",
"environment":"sensu",
"subscriptions":[
"linux",
"dev"
]
}
}
sudo vim /etc/sensu/conf.d/redis.json
{
"redis":{
"host":" redis-server-ip ",
"port":6379,
"reconnect_on_error":true,
"auto_reconnect":true
}
}
sudo vim /etc/sensu/conf.d/transport.json
{
"transport":{
"name":"redis",
"reconnect_on_error":true
}
}
sudo update-rc.d sensu-client defaults
sudo update-rc.d uchiwa defaults
sudo vim /etc/sensu/uchiwa.json
{
"sensu":[
{
"name":"sensu",
"host":" sensu-server-ip or hostname ",
"port":4567,
"timeout":10
}
],
"uchiwa":{
"host":"0.0.0.0",
"port":3000,
"refresh":10
}
}
sudo service sensu-client start
sudo service uchiwa start
Set Up a Check
Add a check on both server
Install sensu-plugins-XXX on all sensu client for check.
sudo sensu-install -p sensu-plugins-network-checks
……
Add check_xxx.json file on only the Sensu master server.
sudo vim /etc/sensu/conf.d/check.json
{
"checks":{
"check_ssh":{
"type":"metric",
"command":"check-ports.rb -h 127.0.0.1 -p 22 -t 30",
"interval":60,
"subscribers":[
"dev"
],
"handlers":[
"logstash"
]
},
"check_dns":{
"type":"metric",
"command":"check-ports.rb -h 127.0.0.1 -p 53 -t 30",
"interval":60,
"subscribers":[
"dev"
],
"handlers":[
"logstash"
]
},
"check_http":{
"type":"metric",
"command":"check-ports.rb -h 127.0.0.1 -p 80 -t 30",
"interval":60,
"subscribers":[
"dev"
],
"handlers":[
"logstash"
]
},
"check_https":{
"type":"metric",
"command":"check-ports.rb -h 127.0.0.1 -p 443 -t 30",
"interval":60,
"subscribers":[
"dev"
],
"handlers":[
"logstash"
]
}
}
}
sudo service sensu-client restart
Set Up a Handler
Add a handler
sudo apt-get install -y build-essential
sudo sensu-install -p sensu-plugins-logstash
On the Sensu master server, create and edit thehandler_logstash.json file.
sudo vim /etc/sensu/conf.d/handlers.json
{
"handlers":{
"logstash":{
"type":"pipe",
"command":"handler-logstash.rb"
}
}
}
sudo vim /etc/sensu/conf.d/handler_logstash.json
{
"logstash" :{
"server" :"127.0.0.1",
"port" :6379,
"list" :"logstash",
"type" :"sensu",
"output" :"redis",
"custom" :{
"thisFieldWillBeMergedIntoTheTopLevelOfOutgoingJSON" :{
"metadata" :"some metadata",
"moreMetadata" :42
}
}
}
}
sudo service sensu-server restart && sudo service sensu-api restart
Logstash
Install Java 8
sudo add-apt-repository -y ppa:webupd8team/java
sudo apt-get update && sudo apt-get -y install oracle-java8-installer
Install logstash
wget -qO - https://artifacts.elastic.co/GPG-KEY-elasticsearch | sudo- apt-key add -
echo "deb https://artifacts.elastic.co/packages/5.x/apt stable main"- | sudo tee -a /etc/apt/sources.list.d/elastic-5.x.list
sudo apt-get update && sudo apt-get install logstash
sudo initctl start logstash
Configure logstash
sudo vim /etc/logstash/conf.d/sensu-input-logstash.conf
input{
redis{
data_type => "list"
key => "logstash"
host => "127.0.0.1"
port => 6379
threads => 5
}
}
sudo vim /etc/logstash/conf.d/elasticsearch-output.conf
output {
elasticsearch {
hosts => [ "localhost:9200" ]
manage_template => false
index => "logstash-%{type}-%{+YYYY.MM.dd}"
document_type => "%{type}"
}
}
sudo initctl start logstash
Elasticsearch
Install elasticsearch
sudo apt-get install elasticsearch
sudo update-rc.d elasticsearch defaults 95 10
sudo /usr/share/elasticsearch/bin/elasticsearch-plugin install analysis-smartcn
sudo -i service elasticsearch start
Change user to root and add “network.host: 0.0.0.0” to
/etc/elasticsearch/elasticsearch.yml
#This file can only be accessed by root and cannot be modified using sudo.
sudo su
Vim /etc/elasticsearch/elasticsearch.yml
network.host: 0.0.0.0
sudo -i service elasticsearch restart
Kibana – install in another instance
Install kibana
wget -qO - https://artifacts.elastic.co/GPG-KEY-elasticsearch | sudo- apt-key add -
echo "deb https://artifacts.elastic.co/packages/5.x/apt stable main" | sudo tee -a /etc/apt/sources.list.d/elastic-5.x.list
sudo apt-get update && sudo apt-get install kibana
Install plugin own_home
sudo /usr/share/kibana/bin/kibana-plugin install https://github.com/wtakase/kibana-own-home/releases/download/v5.2.2/own_home-5.2.2.zip
Configure and run
sudo vim /etc/kibana/kibana.yml
Notice change the follow ‘es-ip’ to ip address which elasticsearch bind.
server.host: "0.0.0.0"
elasticsearch.url: "http://localhost:19200"
elasticsearch.requestHeadersWhitelist: [x-proxy-user, cookie ]
own_home.elasticsearch.url: "http://es-ip:9200"
own_home.session.secretkey: f9e794323b453885f5181f1b624d0a
own_home.session.isSecure: false
own_home.local.groups: [share01, share02]
sudo service kibana start
Setup front end web server (we use nginx) and configure Authentication (only for test)
Install nginx
sudo apt-get -y install nginx
Create user admin
sudo sh -c "echo -n 'admin:' >> /etc/nginx/.htpasswd"
sudo sh -c "openssl passwd -apr1 >> /etc/nginx/.htpasswd"
Create user user01
sudo sh -c "echo -n 'user01:' >> /etc/nginx/.htpasswd"
sudo sh -c "openssl passwd -apr1 >> /etc/nginx/.htpasswd"
Configure Nginx
sudo vim /etc/nginx/sites-available/kibana
server {
listen 80 default_server;
listen [::]:80 default_server ipv6only=on;
server_name localhost(yourdomain.com);
location / {
auth_basic "Restricted Content";
auth_basic_user_file /etc/nginx/.htpasswd;
proxy_pass http://localhost:5601;
proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection 'upgrade';
proxy_set_header Host $host;
proxy_cache_bypass $http_upgrade;
proxy_set_header x-proxy-user $remote_user;
}
}
sudo rm /etc/nginx/sites-enabled/default && sudo ln -s /etc/nginx/sites-available/kibana /etc/nginx/sites-enabled/kibana
Restart nginx
sudo service nginx restart
More flexible configuration
https://github.com/wtakase/kibana-own-home
Grafana –install in the Instance where Kibana was installed
Install grafana
curl https://packagecloud.io/gpg.key | sudo apt-key add -
sudo echo "deb https://packagecloud.io/grafana/stable/debian/ jessie- main" |sudo tee -a /etc/apt/sources.list.d/grafana.list
sudo apt-get update && sudo apt-get -y install grafana
As 3000 port has been used by uchiwa so we change the port default 3000 to 3300. Add “http_port = 3300”.
sudo vim /etc/grafana/grafana.ini
http_port = 3300
sudo service grafana-server start
Open your browser at http://ip:3300 and use admin/admin as the credentials to access Grafana: