日志分析 第六章 安装elasticsearch

在这里，以两台es集群为例。

es集群健康状况有三种状态，这里我们搭建的es集群，只要两台不同时挂掉，数据不会丢失。

green	所有主要分片和复制分片都可用
yellow	所有主要分片可用，但不是所有复制分片都可用
red	不是所有的主要分片都可用

举个例子：

• 比如说现在集群节点es1位主节点，es2位复制分片节点，默认情况下，两台es都接收logstash传过来的日志，是负载均衡的。
• 如果es1宕掉，es2会被提升为主节点，只有es2接收logstash传来得日志数据，同时整个集群状态由green转为red。
• 修复es1，es1会作为分片节点加入集群中，es1加入到集群中后，会进行修复，es2中新增的数据同步到es1分片节点，同步过程整个集群健康状态转为yellow。
• 当同步完成，集群状态转为green。

安装elasticsearch

官网：https://www.elastic.co/products/elasticsearch

# tar xf elasticsearch-2.3..tar.gz -C /usr/local/app/

# ln -sv /usr/local/app/elasticsearch-2.3. /usr/local/elasticsearch

# chown –R nobody.nobody /usr/local/app/elasticsearch-2.3./

# cd /usr/local/elasticsearch

编辑配置文件

es1

cluster.name: myES

node.name: es1

node.master: true

path.data: /data/elasticsearch/data

path.logs: /data/elasticsearch/log

bootstrap.mlockall: True

network.host: 10.80.2.83

http.port:

http.enabled: true

transport.tcp.port:

transport.tcp.compress: true

discovery.zen.ping.unicast.hosts: ["10.80.2.83:9300","10.80.2.84:9300"]

discovery.zen.minimum_master_nodes:

discovery.zen.ping.timeout: 3s

discovery.zen.ping.multicast.enabled: false

gateway.recover_after_nodes:

gateway.recover_after_time: 5m

gateway.expected_nodes:

cluster.routing.allocation.node_initial_primaries_recoveries:

indices.recovery.max_size_per_sec: 50mb

index :

  analysis :

    analyzer :

      default :

        tokenizer : keyword

config/elasticsearch.yml

#!/bin/sh

# check in case a user was using this mechanism

if [ "x$ES_CLASSPATH" != "x" ]; then

    cat >& << EOF

Error: Don't modify the classpath with ES_CLASSPATH. Best is to add

additional elements via the plugin mechanism, or if code must really be

added to the main classpath, add jars to lib/ (unsupported).

EOF

    exit

fi

ES_CLASSPATH="$ES_HOME/lib/elasticsearch-2.3.4.jar:$ES_HOME/lib/*"

if [ "x$ES_MIN_MEM" = "x" ]; then

    ES_MIN_MEM=8g

fi

if [ "x$ES_MAX_MEM" = "x" ]; then

    ES_MAX_MEM=8g

fi

if [ "x$ES_HEAP_SIZE" != "x" ]; then

    ES_MIN_MEM=$ES_HEAP_SIZE

    ES_MAX_MEM=$ES_HEAP_SIZE

fi

# min and max heap sizes should be set to the same value to avoid

# stop-the-world GC pauses during resize, and so that we can lock the

# heap in memory on startup to prevent any of it from being swapped

# out.

JAVA_OPTS="$JAVA_OPTS -Xms${ES_MIN_MEM}"

JAVA_OPTS="$JAVA_OPTS -Xmx${ES_MAX_MEM}"

# new generation

if [ "x$ES_HEAP_NEWSIZE" != "x" ]; then

    JAVA_OPTS="$JAVA_OPTS -Xmn${ES_HEAP_NEWSIZE}"

fi

# max direct memory

if [ "x$ES_DIRECT_SIZE" != "x" ]; then

    JAVA_OPTS="$JAVA_OPTS -XX:MaxDirectMemorySize=${ES_DIRECT_SIZE}"

fi

# set to headless, just in case

JAVA_OPTS="$JAVA_OPTS -Djava.awt.headless=true"

# Force the JVM to use IPv4 stack

if [ "x$ES_USE_IPV4" != "x" ]; then

  JAVA_OPTS="$JAVA_OPTS -Djava.net.preferIPv4Stack=true"

fi

# Add gc options. ES_GC_OPTS is unsupported, for internal testing

if [ "x$ES_GC_OPTS" = "x" ]; then

  ES_GC_OPTS="$ES_GC_OPTS -XX:+UseParNewGC"

  ES_GC_OPTS="$ES_GC_OPTS -XX:+UseConcMarkSweepGC"

  ES_GC_OPTS="$ES_GC_OPTS -XX:CMSInitiatingOccupancyFraction=75"

  ES_GC_OPTS="$ES_GC_OPTS -XX:+UseCMSInitiatingOccupancyOnly"

fi

JAVA_OPTS="$JAVA_OPTS $ES_GC_OPTS"

# GC logging options

if [ -n "$ES_GC_LOG_FILE" ]; then

  JAVA_OPTS="$JAVA_OPTS -XX:+PrintGCDetails"

  JAVA_OPTS="$JAVA_OPTS -XX:+PrintGCTimeStamps"

  JAVA_OPTS="$JAVA_OPTS -XX:+PrintGCDateStamps"

  JAVA_OPTS="$JAVA_OPTS -XX:+PrintClassHistogram"

  JAVA_OPTS="$JAVA_OPTS -XX:+PrintTenuringDistribution"

  JAVA_OPTS="$JAVA_OPTS -XX:+PrintGCApplicationStoppedTime"

  JAVA_OPTS="$JAVA_OPTS -Xloggc:$ES_GC_LOG_FILE"

  # Ensure that the directory for the log file exists: the JVM will not create it.

  mkdir -p "`dirname \"$ES_GC_LOG_FILE\"`"

fi

# Causes the JVM to dump its heap on OutOfMemory.

JAVA_OPTS="$JAVA_OPTS -XX:+HeapDumpOnOutOfMemoryError"

# The path to the heap dump location, note directory must exists and have enough

# space for a full heap dump.

#JAVA_OPTS="$JAVA_OPTS -XX:HeapDumpPath=$ES_HOME/logs/heapdump.hprof"

# Disables explicit GC

JAVA_OPTS="$JAVA_OPTS -XX:+DisableExplicitGC"

# Ensure UTF- encoding by default (e.g. filenames)

JAVA_OPTS="$JAVA_OPTS -Dfile.encoding=UTF-8"

# Use our provided JNA always versus the system one

JAVA_OPTS="$JAVA_OPTS -Djna.nosys=true"

bin/elasticsearch.in.sh

es2

#集群名字

cluster.name: myES

#节点名字

node.name: es2

#是否有资格成为主节点

node.master: true

#数据目录

path.data: /data/elasticsearch/data

#日志目录

path.logs: /data/elasticsearch/log

#锁住内存

bootstrap.mlockall: True

#绑定的IP地址，可为0.0.0.

#network.bind_host: 10.80.2.84

#该节点与其他节点交互的ip地址，

#network.publish_host: 10.80.2.84

#上面两个参数集合

network.host: 10.80.2.84

#对外服务端口

http.port:

#是否使用http协议对外提供服务

http.enabled: true

#节点间交互tcp端口

transport.tcp.port:

#传输数据时压缩

transport.tcp.compress: true

#集群中主节点初始化列表，通过这些节点自动发现加入其他节点

discovery.zen.ping.unicast.hosts: ["10.80.2.83:9300","10.80.2.84:9300"]

#保证集群中节点可以知道其他N各有主节点资格，默认1，es数超过2个，可设置大一些

discovery.zen.minimum_master_nodes:

#自动发现其他节点时ping超时时间，网络环境差，提高此参数

discovery.zen.ping.timeout: 3s

#是否打开多播发现节点，默认true

discovery.zen.ping.multicast.enabled: false

#集群中几个节点启动时进行数据恢复

gateway.recover_after_nodes:

#初始化数据恢复进程超时时间

gateway.recover_after_time: 5m

#这个es集群中节点数量，一旦这些数目节点启动，进行数据恢复

gateway.expected_nodes:

#初始化数据恢复时，并发恢复的线程个数

cluster.routing.allocation.node_initial_primaries_recoveries:

#数据恢复时最大带宽

indices.recovery.max_size_per_sec: 50mb

# 禁止分词，

index :

  analysis :

    analyzer :

      default :

        tokenizer : keyword

config/elasticsearch.yml

#!/bin/sh

# check in case a user was using this mechanism

if [ "x$ES_CLASSPATH" != "x" ]; then

    cat >& << EOF

Error: Don't modify the classpath with ES_CLASSPATH. Best is to add

additional elements via the plugin mechanism, or if code must really be

added to the main classpath, add jars to lib/ (unsupported).

EOF

    exit

fi

ES_CLASSPATH="$ES_HOME/lib/elasticsearch-2.3.4.jar:$ES_HOME/lib/*"

if [ "x$ES_MIN_MEM" = "x" ]; then

    ES_MIN_MEM=8g

fi

if [ "x$ES_MAX_MEM" = "x" ]; then

    ES_MAX_MEM=8g

fi

if [ "x$ES_HEAP_SIZE" != "x" ]; then

    ES_MIN_MEM=$ES_HEAP_SIZE

    ES_MAX_MEM=$ES_HEAP_SIZE

fi

# min and max heap sizes should be set to the same value to avoid

# stop-the-world GC pauses during resize, and so that we can lock the

# heap in memory on startup to prevent any of it from being swapped

# out.

JAVA_OPTS="$JAVA_OPTS -Xms${ES_MIN_MEM}"

JAVA_OPTS="$JAVA_OPTS -Xmx${ES_MAX_MEM}"

# new generation

if [ "x$ES_HEAP_NEWSIZE" != "x" ]; then

    JAVA_OPTS="$JAVA_OPTS -Xmn${ES_HEAP_NEWSIZE}"

fi

# max direct memory

if [ "x$ES_DIRECT_SIZE" != "x" ]; then

    JAVA_OPTS="$JAVA_OPTS -XX:MaxDirectMemorySize=${ES_DIRECT_SIZE}"

fi

# set to headless, just in case

JAVA_OPTS="$JAVA_OPTS -Djava.awt.headless=true"

# Force the JVM to use IPv4 stack

if [ "x$ES_USE_IPV4" != "x" ]; then

  JAVA_OPTS="$JAVA_OPTS -Djava.net.preferIPv4Stack=true"

fi

# Add gc options. ES_GC_OPTS is unsupported, for internal testing

if [ "x$ES_GC_OPTS" = "x" ]; then

  ES_GC_OPTS="$ES_GC_OPTS -XX:+UseParNewGC"

  ES_GC_OPTS="$ES_GC_OPTS -XX:+UseConcMarkSweepGC"

  ES_GC_OPTS="$ES_GC_OPTS -XX:CMSInitiatingOccupancyFraction=75"

  ES_GC_OPTS="$ES_GC_OPTS -XX:+UseCMSInitiatingOccupancyOnly"

fi

JAVA_OPTS="$JAVA_OPTS $ES_GC_OPTS"

# GC logging options

if [ -n "$ES_GC_LOG_FILE" ]; then

  JAVA_OPTS="$JAVA_OPTS -XX:+PrintGCDetails"

  JAVA_OPTS="$JAVA_OPTS -XX:+PrintGCTimeStamps"

  JAVA_OPTS="$JAVA_OPTS -XX:+PrintGCDateStamps"

  JAVA_OPTS="$JAVA_OPTS -XX:+PrintClassHistogram"

  JAVA_OPTS="$JAVA_OPTS -XX:+PrintTenuringDistribution"

  JAVA_OPTS="$JAVA_OPTS -XX:+PrintGCApplicationStoppedTime"

  JAVA_OPTS="$JAVA_OPTS -Xloggc:$ES_GC_LOG_FILE"

  # Ensure that the directory for the log file exists: the JVM will not create it.

  mkdir -p "`dirname \"$ES_GC_LOG_FILE\"`"

fi

# Causes the JVM to dump its heap on OutOfMemory.

JAVA_OPTS="$JAVA_OPTS -XX:+HeapDumpOnOutOfMemoryError"

# The path to the heap dump location, note directory must exists and have enough

# space for a full heap dump.

#JAVA_OPTS="$JAVA_OPTS -XX:HeapDumpPath=$ES_HOME/logs/heapdump.hprof"

# Disables explicit GC

JAVA_OPTS="$JAVA_OPTS -XX:+DisableExplicitGC"

# Ensure UTF- encoding by default (e.g. filenames)

JAVA_OPTS="$JAVA_OPTS -Dfile.encoding=UTF-8"

# Use our provided JNA always versus the system one

JAVA_OPTS="$JAVA_OPTS -Djna.nosys=true"

bin/elasticsearch.in.sh