diff --git a/elasticsearch-heat.yaml b/elasticsearch-heat.yaml index c6e9af47eedd2f8136d2ab1293fde9e1f23c59ed..473bdbd62343c7688739c27cf82c63b942c2df6a 100644 --- a/elasticsearch-heat.yaml +++ b/elasticsearch-heat.yaml @@ -23,7 +23,7 @@ parameters: es_data_nodes: type: number description: Number of master nodes for the ES cluster - default: 3 + default: 2 constraints: - range: {min: 1, max: 10} es_data_flavor: @@ -32,6 +32,10 @@ parameters: default: m1.medium constraints: - allowed_values: [ m1.small, m1.medium, m1.large ] + image_name: + type: string + description: Image used for the machines + default: "CC7 Extra - x86_64 [2015-02-10]" resources: master_nodes: type: OS::Heat::ResourceGroup @@ -46,7 +50,7 @@ resources: params: $cluster_name: {get_param: cluster_name} key_name: testing - image: "CC7 Extra - x86_64 [2015-02-10]" + image: {get_param: image_name} flavor: {get_param: es_master_flavor} user_data: str_replace: @@ -69,7 +73,7 @@ resources: params: $cluster_name: {get_param: cluster_name} key_name: testing - image: "CC7 Extra - x86_64 [2015-02-10]" + image: {get_param: image_name} flavor: {get_param: es_data_flavor} user_data: str_replace: @@ -79,3 +83,46 @@ resources: $master_node: "false" $data_node: "true" template: {get_file: es_init.sh} +outputs: + masters: + description: The name of the created masters + value: {get_attr: [master_nodes, name]} + data_nodes: + description: The name of the created data nodes + value: {get_attr: [data_nodes, name]} + kibana_url: + description: The kibana URL. Access it through a web browser + value: + str_replace: + template: "http://%name%.cern.ch:5601" + params: + "%name%": {get_attr: [master_nodes, name, 0]} + es_url: + description: The ES URL. Do `curl http://<url>/_cat/nodes` to see all the + nodes of the cluster. + value: + str_replace: + template: "http://%name%.cern.ch:9200" + params: + "%name%": {get_attr: [master_nodes, name, 0]} + elastichq_url: + description: URL for the elasticHQ monitoring plugin + value: + str_replace: + template: "http://%name%.cern.ch:9200/_plugin/HQ/?url=http://%name%.cern.ch:9200" + params: + "%name%": {get_attr: [master_nodes, name, 0]} + head_url: + description: URL for the head plugin + value: + str_replace: + template: "http://%name%.cern.ch:9200/_plugin/head/" + params: + "%name%": {get_attr: [master_nodes, name, 0]} + kopf_url: + description: URL for the Kopf plugin + value: + str_replace: + template: "http://%name%.cern.ch:9200/_plugin/kopf/" + params: + "%name%": {get_attr: [master_nodes, name, 0]} diff --git a/manifests/datanode.pp b/manifests/datanode.pp index c70ad7fc809f858343a565aee4c22f5530bbaa3e..4238379cacae3b7f64de06365d24615ac258a1ff 100644 --- a/manifests/datanode.pp +++ b/manifests/datanode.pp @@ -8,4 +8,16 @@ class datanode { config => merge(hiera_hash('master_config'),hiera_hash('datanode_config')), } + flume::tail { "tail-es-$::hostgroup_2": # give a name without spaces otherwise it will fail + files => ["/var/log/elasticsearch/$::hostgroup_2.log"], # your list of files to tail + out => { + 'es1' => { + type => 'es', + hostNames => "$::hostgroup_2-master-0.cern.ch,$::hostgroup_2-master-1.cern.ch", # coma separated list of ES nodes with no spaces in-between + indexName => 'logs', # name of the index in ES + clusterName => $::hostgroup_2, # name of your ES cluster + } + } + } + } diff --git a/manifests/master.pp b/manifests/master.pp index e74a192c4d2bc0118a9a66bef7fb1073854ceaf5..8b26e20f5c01f00b44cfa0665d0a33c48aed0a97 100644 --- a/manifests/master.pp +++ b/manifests/master.pp @@ -17,4 +17,37 @@ class master { module_dir => 'kopf'; } + # Kibana 4 + + $kibana_version = 'kibana-4.0.2-linux-x64' + $kibana_link = "https://download.elasticsearch.org/kibana/kibana/${kibana_version}.tar.gz" + + ensure_packages('wget') + + exec { 'install_kibana4': + command => "/usr/bin/wget -P /tmp ${kibana_link} && /bin/tar -xvzf /tmp/${kibana_version}.tar.gz -C /usr/share -o", + creates => "/usr/share/${kibana_version}", + require => Package['wget'] + } -> + file { '/etc/systemd/system/kibana4.service': + content => template('/etc/puppet/templates/elasticsearch/kibana4.service.erb') + } -> + service { 'kibana4': + ensure => running, + hasstatus => true, + hasrestart => true, + enable => true, + } + + flume::tail { "tail-es-$::hostgroup_2": # give a name without spaces otherwise it will fail + files => ["/var/log/elasticsearch/$::hostgroup_2.log"], # your list of files to tail + out => { + 'es1' => { + type => 'es', + hostNames => "$::hostgroup_2-master-0.cern.ch,$::hostgroup_2-master-1.cern.ch", # coma separated list of ES nodes with no spaces in-between + indexName => 'logs', # name of the index in ES + clusterName => $::hostgroup_2, # name of your ES cluster + } + } + } } diff --git a/templates/elasticsearch/kibana4.service.erb b/templates/elasticsearch/kibana4.service.erb new file mode 100644 index 0000000000000000000000000000000000000000..9df36ded55abd6d665a15bd89270822a326daa1e --- /dev/null +++ b/templates/elasticsearch/kibana4.service.erb @@ -0,0 +1,12 @@ +[Service] +ExecStart=/usr/share/<%= @kibana_version %>/bin/kibana +Restart=always +StandardOutput=syslog +StandardError=syslog +SyslogIdentifier=kibana4 +User=root +Group=root +Environment=NODE_ENV=production + +[Install] +WantedBy=multi-user.target