run.py

#!/usr/bin/env python

# Copyright (C) 2013-2014 SignalFuse, Inc.

# Start script for ElasticSearch.
# Requires python-yaml for configuration writing.

import os
import yaml

from maestro.guestutils import *

ELASTICSEARCH_CONFIG_FILE = os.path.join('config', 'elasticsearch.yml')
ELASTICSEARCH_LOGGING_CONFIG = os.path.join('config', 'logging.yml')
DEFAULT_ELASTICSEARCH_ZONE = 'ether'
LOG_PATTERN = "%d{yyyy'-'MM'-'dd'T'HH:mm:ss.SSSXXX} %-5p [%-35.35t] [%-36.36c]: %m%n"

os.chdir(os.path.join(
    os.path.dirname(os.path.abspath(__file__)),
    '..'))

def env_as_bool(k, default=True):
    if k in os.environ:
        v = os.environ[k].lower()
        return v == 'true'
    return default

# Prepare the YAML configuration and write it.
with open(ELASTICSEARCH_CONFIG_FILE, 'w+') as conf:
    data = {
        'cluster': {
            'name': os.environ.get('CLUSTER_NAME',
                                   '{}-elasticsearch'.format(get_environment_name())),
            'routing.allocation.awareness.attributes': 'zone',
        },

        # Node configuration
        'node': {
            'name': get_container_name(),
            'zone': os.environ.get('ZONE_NAME', DEFAULT_ELASTICSEARCH_ZONE),
            'data': env_as_bool('IS_DATA_NODE'),
            'master': env_as_bool('IS_MASTER_NODE'),
        },

        # Index/replica configuration
        'index': {
            'number_of_replicas': int(os.environ.get('NUM_INDEX_REPLICAS', 1)),
            'number_of_shards': int(os.environ.get('NUM_INDEX_SHARDS', 5)),
        },

        'path': {
            'data': os.environ.get('PATH_DATA', '/var/lib/elasticsearch').split(','),
            'logs': '/var/log/elasticsearch',
        },

        'transport.tcp.port': get_port('peer', 9300),

        'http': {
            # HTTP is disabled iff the node is *only* a data node.
            'enabled': not (env_as_bool('IS_DATA_NODE') and
                            not env_as_bool('IS_MASTER_NODE')),
            'port': get_port('http', 9200),
        },

        # Network and discovery.
        'network': {
            'publish_host': get_container_host_address(),
        },
        'discovery': {
                'type': 'com.sonian.elasticsearch.zookeeper.discovery.ZooKeeperDiscoveryModule',
                'zen.multicast.enabled': False,
        },
        'sonian.elasticsearch.zookeeper': {
                'settings.enabled': False,
                'client.host': ','.join(get_node_list('zookeeper', ports=['client'])),
                'discovery.state_publishing.enabled': True,
        },
        'zookeeper.root': os.environ.get('ZOOKEEPER_BASE',
                                         '/{}/elasticsearch'.format(get_environment_name())),

        # Marvel plugin configuration.
        'marvel': {
            'agent': {
                'enabled': env_as_bool('MARVEL_ENABLED'),
                # TODO: improve this, ideally we want to figure this out
                # automatically.
                'exporter.es.hosts': \
                    os.environ.get('MARVEL_TARGETS', 'localhost:9200').split(','),
            },
        },

        # AWS Cloud plugin configuration.
        'cloud': {
            'aws': {
                'region': os.environ.get('ZONE_NAME', DEFAULT_ELASTICSEARCH_ZONE),
            },
        },
    }

    if os.environ.get('AWS_ACCESS_KEY') and os.environ.get('AWS_SECRET_KEY'):
        data['cloud']['aws'].update({
            'access_key': os.environ['AWS_ACCESS_KEY'],
            'secret_key': os.environ['AWS_SECRET_KEY'],
        })

    yaml.dump(data, conf, default_flow_style=False)

# Setup the logging configuration
with open(ELASTICSEARCH_LOGGING_CONFIG, 'w+') as conf:
    yaml.dump({
        'es.logger.level': 'INFO',
        'rootLogger': '${es.logger.level},R',
        'logger': {
            'action': 'DEBUG',
            'com.amazonaws': 'WARN',
        },
        'appender': {
            'R': {
                'type': 'rollingFile',
                'File': '${path.logs}/%s.log' % get_container_name(),
                'MaxFileSize': '100MB',
                'MaxBackupIndex': '10',
                'layout': {
                  'type': 'pattern',
                  'ConversionPattern': LOG_PATTERN,
                },
            },
        },
    }, conf, default_flow_style=False)

# Start ElasticSearch
os.execl('bin/elasticsearch', 'elasticsearch')