#!/usr/bin/env python
from __future__ import print_function

import ConfigParser
import StringIO
import argparse
import cookielib
import datetime
import httplib
import json
import logging
import os
import pwd
import socket
import subprocess
import sys
import time
import traceback
import urllib
import urllib2
import urlparse


_logger = logging.getLogger('rsw-installer')


DATA_SECTION = "DATASOURCE"
DATA_URL_OPTION = "datasource.url"
DATA_UNAME_OPTION = "datasource.username"
DATA_PWD_OPTION = "datasource.password"


def dbg(*args, **kwargs):
    buf = StringIO.StringIO()
    kwargs['file'] = buf
    print(*args, **kwargs)
    _logger.debug(buf.getvalue().rstrip('\n'))
    buf.close()


def log(*args, **kwargs):
    buf = StringIO.StringIO()
    kwargs['file'] = buf
    print(*args, **kwargs)
    _logger.info(buf.getvalue().rstrip('\n'))
    buf.close()


def check_call_with_logging(*args, **kwargs):
    """Wraps subprocess.check_call so that the output gets logged to both log file
    and to console.
    """
    kwargs['stdout'] = subprocess.PIPE
    kwargs['stderr'] = subprocess.PIPE
    proc = subprocess.Popen(*args, **kwargs)

    def log_stream_lines(stream):
        for line in iter(stream.readline, b''):
            log(line)

    while proc.poll() is None:
        log_stream_lines(proc.stdout)
        log_stream_lines(proc.stderr)
        time.sleep(0.01)

    # Handle lines possibly left in the buffers
    log_stream_lines(proc.stdout)
    log_stream_lines(proc.stderr)

    status = proc.returncode
    if status != 0:
        raise subprocess.CalledProcessError(proc.returncode, cmd=args)
    return proc


def call_with_logging(*args, **kwargs):
    """Wraps subprocess.call so that the output gets logged to both log file and
    to console.
    """
    try:
        proc = check_call_with_logging(*args, **kwargs)
        return proc.returncode
    except subprocess.CalledProcessError, cpe:
        return cpe.returncode


def call_dropping_return_code(*args, **kwargs):
    """Returns the output of subprocess.Popen.communicate. A convenience function
    when all we want is the output and we don't care if the process failed or
    not.
    """
    kwargs["stdout"] = subprocess.PIPE
    kwargs["stderr"] = subprocess.PIPE
    p = subprocess.Popen(*args, **kwargs)
    return p.communicate()


class StepFailed(Exception):
    pass


class Step(object):
    def __init__(self, prerequisite=False, help_message=None):
        self.prerequisite = prerequisite
        self.help_message = help_message

    def describe(self, what):
        log(u"# {0}".format(self.description))
        if what is not None:
            log(u"#  {0}".format(what))
        log(u"##")


class ShellCommandStep(Step):
    def __init__(self, command, description, shell=False, *args, **kwargs):
        super(ShellCommandStep, self).__init__(*args, **kwargs)
        self.command = command
        self.description = description
        self.shell = shell

    def dry_run(self):
        self.describe('`' + self.printable_command() + '`')

    def printable_command(self):
        if isinstance(self.command, basestring):
            return self.command
        else:
            return " ".join(self.command)

    def __call__(self):
        self.dry_run()
        proc = subprocess.Popen(self.command,
                                stdout=subprocess.PIPE, stderr=subprocess.PIPE,
                                shell=self.shell)

        def log_stream_lines(stream):
            for line in iter(stream.readline, b''):
                log(line)

        while proc.poll() is None:
            log_stream_lines(proc.stdout)
            log_stream_lines(proc.stderr)
            time.sleep(0.01)

        # Handle lines possibly left in the buffers
        log_stream_lines(proc.stdout)
        log_stream_lines(proc.stderr)

        status = proc.returncode
        if status != 0:
            raise StepFailed(
                u"Command `{0}` failed with status code {1}".format(
                    self.printable_command(), status))


class OutputSuppressingShellCommandStep(ShellCommandStep):
    def __init__(self, command, suppressed_tokens, *args, **kwargs):
        args = command + list(args) # otherwise suppressed_tokens will be interpreted as the command
        super(OutputSuppressingShellCommandStep, self).__init__(*args, **kwargs)
        self.tokens = suppressed_tokens

    def __call__(self):
        self.dry_run()
        proc = subprocess.Popen(self.command,
                                stdout=subprocess.PIPE, stderr=subprocess.PIPE,
                                shell=self.shell)
        suppressed_lines = []

        def milk_suppressed_lines(stream):
            result = []

            for line in iter(stream.readline, b''):
                suppressed = any([token in line for token in self.tokens])
                if not suppressed:
                    log(line)
                else:
                    log(u".")
                    result.append(line)

            return result

        while proc.poll() is None:
            suppressed_lines += milk_suppressed_lines(proc.stdout)
            suppressed_lines += milk_suppressed_lines(proc.stderr)
            time.sleep(0.01)

        # Handle the lines possibly left in the buffers
        suppressed_lines += milk_suppressed_lines(proc.stdout)
        suppressed_lines += milk_suppressed_lines(proc.stderr)

        status = proc.returncode
        if status != 0:
            log("# Suppressed output lines for command:")
            log("#  (not always in order due to the complexity of stdout/stderr output handling)")
            for line in suppressed_lines:
                log(line)

            raise StepFailed(
                u"Command `{0}` failed with status code {1}".format(
                    " ".join(self.printable_command()), status))


class FunctionStep(Step):
    def __init__(self, func, description, *args, **kwargs):
        super(FunctionStep, self).__init__(*args, **kwargs)
        self.func = func
        self.description = description

    def dry_run(self):
        self.describe(None)

    def __call__(self):
        self.describe(None)
        return self.func()


class RepoState(object):
    def __init__(self, online_allowed):
        self.online = online_allowed
        self.offline = not online_allowed

        self.repo_ok = False
        self.disabled_repos = []


# Module / program level variable, initialized in main.
# Contains shared state by Yum repo steps.
REPO_STATE = None


HOSTNAME_RESOLVABLE_STEP = ShellCommandStep(
    "getent hosts `hostname` > /dev/null",
    "Ensure we can resolve this computer's hostname using either /etc/hosts or DNS",
    shell=True, help_message=
"""If you cannot add your installation host to your network's DNS system, you
can add the hostname to /etc/hosts file. See `man hosts` for more information.

The installation will not be able to run database migration steps if system's
hostname is not resolvable via either DNS or from /etc/hosts.

Some useful commands:
 - getent hosts | grep `hostname`
 - host `hostname`
 - echo "127.0.0.1   `hostname`" >> /etc/hosts""", prerequisite=True)


def _resolve_repos():
    """Returns a list of repo ids."""
    result = []
    repo_line = False
    (repolist_out, repolist_err) = call_dropping_return_code(["yum", "repolist"])
    for line in (repolist_out + repolist_err).split("\n"):
        if line.startswith("repo id"):
            repo_line = True
            continue
        elif line.startswith("repolist:"):
            repo_line = False

        if repo_line is False:
            continue

        repo_id, rest = line.split(" ", 1)
        result.append(repo_id)

    return result


def _base_repo_available():
    assert REPO_STATE is not None

    if REPO_STATE.online:
        try:
            check_call_with_logging(["yum", "clean", "expire-cache"])
            check_call_with_logging(["yum", "makecache"])
        except Exception, e:
            raise StepFailed(u"Could not refresh repository caches: {}".format(e))

        for repo_id in _resolve_repos():
            if "base" in repo_id:
                log(u"Suitable repo found: '{}'".format(repo_id))
                REPO_STATE.repo_ok = True
                return True
        raise StepFailed("No online base repo found.")
    elif REPO_STATE.offline:
        log("Trying to find DVD/USB base repo...")
        try:
            check_call_with_logging(["rsw-rpm-centosbase-repo-create", "-c"])
            log("Suitable offline DVD/USB base repo found.")
        except Exception, e:
            raise StepFailed(
                u"Could not find an available offline CentOS base package repository: {}".format(
                    str(e)))


def _base_repo_setup():
    assert REPO_STATE is not None

    if REPO_STATE.repo_ok:
        return True

    if REPO_STATE.online:
        current_repos = _resolve_repos()
        for repo in REPO_STATE.initial_repos:
            if repo["is-base-repo"] and repo["is-current"]:
                log("Suitable repo found:")
                _log_repos([repo])
                REPO_STATE.repo_ok = True
                return True
        raise StepFailed("Couldn't find base repo for online installation")
    elif REPO_STATE.offline:
        if call_with_logging(["rsw-rpm-centosbase-repo-create", "-c"]) != 0:
            raise StepFailed("No offline repo found")

        def disable_by_id(repo_id):
            if '/' in repo_id:
                repo_id = repo_id.split('/', 1)[0]
            subprocess.check_output(["yum-config-manager", "--disable", repo_id]) # discard output
            REPO_STATE.disabled_repos.append(repo_id)
            log(u"Disabled repo by id '{}'.".format(repo_id))

        for repo_id in _resolve_repos():
            try:
                disable_by_id(repo_id)
            except subprocess.CalledProcessError, cpe:
                raise StepFailed(str(cpe))

        try:
            check_call_with_logging(["rsw-rpm-centosbase-repo-create"])
            check_call_with_logging(["yum", "clean", "expire-cache"])
            check_call_with_logging(["yum", "makecache"])
            log("Added offline base repo.")
        except Exception, e:
            raise StepFailed(str(e))


def _base_repo_cleanup():
    assert REPO_STATE is not None

    # Create a copy of the original list so we can modify the original on the fly.
    for repo_id in list(REPO_STATE.disabled_repos):
        try:
            if '/' in repo_id:
                repo_id = repo_id.split('/', 1)[0]
            subprocess.check_output(["yum-config-manager", "--enable", repo_id]) # discard output
            REPO_STATE.disabled_repos.remove(repo_id)
            log(u"Enabled repo by id {}".format(repo_id))
        except subprocess.CalledProcessError, cpe:
            raise StepFailed(str(cpe))

    try:
        base_repo_file = "/etc/yum.repos.d/base-offline.repo"
        radarsw_repo_file = "/etc/yum.repos.d/radarsw.repo"
        if os.path.exists(base_repo_file):
            os.unlink(base_repo_file)
            log("Removed automatically added base repo.")
        if os.path.exists(radarsw_repo_file):
            os.unlink(radarsw_repo_file)
            log("Removed automatically added radarsw repo.")
    except Exception, e:
        raise StepFailed(str(e))


BASE_REPO_AVAILABLE_STEP = FunctionStep(
    _base_repo_available, "Ensure CentOS base repository is available",
    help_message=
"""For offline installation: have you plugged in the USB stick or DVD? To
mount it, you need to open it in the graphical user interface (or mount it by
hand using the mount command).

For online installation: Is this computer connected to the internet and has
CentOS base Yum repository available and enabled?

The installation installs a number of packages, which require some packages
from CentOS base repository to be available. You may skip this step if you
know the packages will be installable for Yum via some other repository you
have set up.

Some useful commands:
 - yum repolist -v
 - yum-config-manager --enable/--disable <repo-id>""", prerequisite=True)

BASE_REPO_SETUP_STEP = FunctionStep(_base_repo_setup,
    "Take CentOS base repository into use if needed") # TODO: help_message

BASE_REPO_CLEANUP_STEP = FunctionStep(_base_repo_cleanup,
    "Reset Yum repositories to its original state") # TODO: help_message


def resolve_webapp_version():
    args = "rpm -qa --qf '%{NAME} %{VERSION}\n' | grep vaisala-radarsw-webapp | awk -F ' ' '{print $2}'"
    output = subprocess.check_output(['bash', '-c', args])
    output = output.strip()
    return output


def extract_iris_host(file_path):
    """Extracts the data source password from a file and does no error
    handling. User is expected to handle ConfigParser thrown errors.

    See https://docs.python.org/2.7/library/configparser.html#ConfigParser.Error
    for more information.
    """
    config = ConfigParser.SafeConfigParser()
    config.read(file_path)
    return config.get("IRIS_SOCKET_SERVER", "iris.socket.server.host")


def is_iris_host_set(file_path):
    try:
        iris_host = extract_iris_host(file_path)
        if iris_host is not None:
            return True
    except ConfigParser.Error:
        pass

    return False


def resolve_socket_server_address(socket_server_arg, parser, config_file, config_file_rpmsave):
    log(u"## Setup step")
    log(u"# Determining IRIS Socket Server address")
    log(u"##")
    if socket_server_arg is not None:
        return socket_server_arg

    if is_iris_host_set(config_file):
        from_configuration = extract_iris_host(config_file)
        log(u"Using the value found from the configuration file ({0}): {1}".format(
            config_file, from_configuration))
        return from_configuration
    elif is_iris_host_set(config_file_rpmsave):
        from_rpmsave = extract_iris_host(config_file_rpmsave)
        log(u"Using the value found from the .rpmsave file ({0}): {1}".format(
            config_file_rpmsave, from_rpmsave))
        return from_rpmsave
    else:
        parser.error(u"Cannot determine IRIS Socket Server address. Missing -s option?")


def socket_server_version_check(host):
    SSERVER_PORT = 30735

    def make_request(command, params={}):
        param_parts = []
        for k, v in params.iteritems():
            param_parts.append(u"{0}={1}".format(k, v))
        params = "&".join(param_parts)

        length = len(params) + 1 + len(command)
        return str(length).zfill(9) + command + '|' + params

    VERSION_REQUEST = make_request("SITE_VERSION")

    def read_bytes(sock, count):
        bytes_to_read = count
        bytes_received = 0
        buf = []
        while bytes_received < bytes_to_read:
            received = sock.recv(bytes_to_read - bytes_received)
            if received and len(received) > 0:
                buf.append(received)
                bytes_received += len(received)
        return ''.join(buf)

    def read_response_length(sock):
        return int(read_bytes(sock, 9))

    def query_version(sock):
        sock.send(VERSION_REQUEST)
        length = read_response_length(sock)
        reply = read_bytes(sock, length)

        ack, payload = reply.split('|')
        payload_parts = payload.split(',')
        version = None
        for part in payload_parts:
            if 'version' in part.lower():
                version = part.split('=')[-1]
        if version is None:
            raise StepFailed(u"Could not parse version from {0}".format(reply))
        return version

    try:
        sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        sock.settimeout(5)
        sock.connect((host, SSERVER_PORT))
        version =  query_version(sock)
    except Exception, e:
        if "Name or service not known" in str(e):
            raise StepFailed(u"Could not resolve IRIS Socket server host name {0}".format(host))
        elif "timed out" in str(e):
            raise StepFailed(u"Could not receive anything from IRIS Socket server with host name {0}. Is the host running a socket server?".format(host))

        raise StepFailed(u"Could not connect to host {0}, error:\n{1}".format(
            host, traceback.format_exc()))

    log(version)
    if len(version) < 2:
        raise StepFailed("Couldn't figure out IRIS Socket server version")


def install_packages_command():
    packages = [
        'vaisala-radarsw-webapp',
        'vaisala-radarsw-webapp-proxy',
        'vaisala-radarsw-monitoring',
        'vaisala-radarsw-backup',
        'vaisala-radarsw-scan-service',
    ]
    return ['yum', '-y', 'install'] + packages


def configure_iris_host_command(socket_server):
    iris_host_command = ["/usr/vaisala/radarsw/configuration/bin/configure-iris-host"]
    if socket_server is not None:
        iris_host_command = iris_host_command + [socket_server]
    return iris_host_command


def wait_until_code(code, netloc, path, target, timeout_secs=90, attempt_delay_secs=1):
    log(u"Waiting for {0} to be available...".format(target))
    start = datetime.datetime.now()
    timeout_delta = datetime.timedelta(seconds=timeout_secs)
    while True:
        try:
            conn = httplib.HTTPConnection(netloc, timeout=1)
            conn.request("HEAD", path)
            res = conn.getresponse()

            if res.status == code:
                now = datetime.datetime.now()
                waited = (now - start).total_seconds()
                log(u"\n{0} up in {1} seconds.".format(target, waited))
                break
            else:
                text_status = str(res.status)
                if len(text_status) > 0:
                    dbg(res.status)
                    sys.stderr.write(text_status[0])
                    sys.stderr.flush()
                else:
                    dbg(res.status)
                    sys.stderr.write("?")
                    sys.stderr.flush()
        except Exception, e:
            dbg(e)
            sys.stderr.write("X")
            sys.stderr.flush()

        now = datetime.datetime.now()
        if (start + timeout_delta) < now:
            waited = (now - start).total_seconds()
            sys.stderr.write('\n')
            sys.stderr.flush()
            raise StepFailed(
                u"Timed out waiting for {0} to start up, waited for {1} seconds.".format(target, waited))

        time.sleep(attempt_delay_secs)


def set_env_for_c5_data_migration(config_file, config_file_rpmsave):
    # Export db settings retrieved from old vsoweb-override.ini file (.rpmsave),
    # import settings are from the newly installed file.  This must be done after rpm install.
    # Required props:
    # datasource.driverClassName=org.postgresql.Driver
    # export.url=jdbc:postgresql://localhost:5432/vsowebdb
    # export.username=vsowebuser
    # export.password=xxx
    # import.url=jdbc:postgresql://localhost:5432/wxdb2
    # import.username=wxuser
    # import.password=xxx
    set_in_env("datasource.driverClassName", "org.postgresql.Driver")
    config = ConfigParser.SafeConfigParser()
    dbg("Reading config file {0}".format(config_file))
    config.read(config_file)
    set_in_env("import.url", config.get(DATA_SECTION, DATA_URL_OPTION))
    set_in_env("import.username", config.get(DATA_SECTION, DATA_UNAME_OPTION))
    set_in_env("import.password", config.get(DATA_SECTION, DATA_PWD_OPTION))
    dbg("Reading config file {0}".format(config_file_rpmsave))
    config.read(config_file_rpmsave)
    set_in_env("export.url", config.get(DATA_SECTION, DATA_URL_OPTION))
    set_in_env("export.username", config.get(DATA_SECTION, DATA_UNAME_OPTION))
    set_in_env("export.password", config.get(DATA_SECTION, DATA_PWD_OPTION))


def set_in_env(name, value):
    if "pwd" in name or "password" in name:
        dbg("Setting in env: {0} = {1}".format(name, value[:2] + "*****"))
    else:
        dbg("Setting in env: {0} = {1}".format(name, value))
    os.environ[name] = value


def file_exists(file):
    dbg(u"Checking whether '{0}' exists.".format(file))
    if not os.path.exists(file):
        raise StepFailed(u"Path '{0}' doesn't exist!".format(file))
    log(u"Path '{}' exists.".format(file))


def validate_terrain_dir(terrain_dir):
    file_exists(terrain_dir)
    shade_dir = os.path.join(terrain_dir, "shade_pyramid")
    slope_dir = os.path.join(terrain_dir, "slope_pyramid")
    if not os.path.exists(shade_dir) or not os.path.exists(slope_dir):
        raise StepFailed(u"Could not locate subdirectories shade_pyramid and slope_pyramid in '{0}'. Check that you are pointing to a valid terrain directory.".format(terrain_dir))
    log(u"Path '{0}' points to a valid terrain directory.".format(terrain_dir))


def validate_gis_dump(gis_dump_path):
    file_exists(gis_dump_path)
    if os.path.isdir(gis_dump_path):
        md5sums_file = os.path.join(gis_dump_path, "md5sums.txt")
        if not os.path.exists(md5sums_file):
            raise StepFailed(u"Could not locate file md5sums.txt in '{0}'. Check that you are pointing to a valid GIS dump directory.".format(gis_dump_path))
        log(u"Path '{0}' points to a valid GIS dump directory.".format(gis_dump_path))
    else:
        log(u"Path '{0}' points to a valid GIS dump file.".format(gis_dump_path))


def make_remove_postgres_92_packages_step():
    non_empty_lines = [line.strip()
             for line in
             subprocess.check_output("rpm -qa --qf '%{NAME}\t%{VERSION}\n'",
                                     shell=True).split("\n")
             if len(line.strip()) > 0]

    postgres_lines = [line for ilen in non_empty_lines if 'postgres' in line]
    package_version_pairs = [line.split("\t") for line in postgres_lines]

    postgres_92_packages = []
    for package_name, version in package_version_pairs:
        if '9.2' in version:
            postgres_92_packages.append(package_name)

    if len(postgres_92_packages) > 0:
        return ShellCommandStep(["yum", "-y", "remove"] + postgres_92_packages,
                                u"Remove PostgreSQL 9.2 packages")
    else:
        return FunctionStep(lambda: log("No PostgreSQL packages to remove."),
                            u"Remove PostgreSQL 9.2 packages")


def can_log_in_as_admin(password):
    LOGIN_URL = "http://127.0.0.1:34080" + "/login"
    cookie_jar = cookielib.CookieJar()

    def fail():
        raise StepFailed(u"Could not authenticate as user 'admin' with given password. Try set admin password with --admin-password parameter.")

    try:
        handlers = [urllib2.HTTPHandler(), urllib2.HTTPCookieProcessor(cookie_jar)]
        opener = urllib2.build_opener(*handlers)
        data = {'username': 'admin', 'password': password}
        response = opener.open(LOGIN_URL, urllib.urlencode(data)).read()
        content = json.loads(response)
        if content['authenticated'] is True:
            log("Succesfully authenticated as user 'admin'.")
            return

        log(repr(content))
        fail()
    except Exception, e:
        log("Login as user 'admin' failed.")
        log(traceback.format_exc())
        fail()


def skip_retry_fail():
    allowed_inputs = ['skip', 'retry', 'fail']

    while True:
        log('Skip, retry or fail step (skip/retry/fail)?')
        reply = raw_input().replace('\n','').strip()
        if reply not in allowed_inputs:
            log(u"Allowed inputs are {0}.".format(", ".join(allowed_inputs)))
        else:
            break

    return reply


def make_gis_dump_existence_check_step(gis_dump_path):
    return FunctionStep(lambda: validate_gis_dump(gis_dump_path),
                        u"Check that GIS dump exists at '{}'".format(gis_dump_path),
                        help_message=
"""Check that GIS dump directory (or file) '{}' points to a valid location.
If you are pointing to a directory, it should contain the md5sums.txt file.
""".format(gis_dump_path), prerequisite=True)


def make_terrain_dir_existence_check_step(terrain_dir):
    return FunctionStep(lambda: validate_terrain_dir(terrain_dir),
                        u"Check that terrain directory '{}' points to a valid location".format(terrain_dir),
                        help_message=
"""Check that terrain directory '{}' points to a valid location.
The terrain directory should contain subdirectories named 'shade_pyramid' and 'slope_pyramid'.
""".format(terrain_dir), prerequisite=True)


def make_verify_executable_is_callable_step(executable):
    return ShellCommandStep(
        [executable, "--test"],
        u'Verify \'{}\' is callable (on PATH; try `export PATH=".:$PATH"`)'.format(
            executable),
        prerequisite=True)


def create_basemap_installation_steps(gis_db_dump, max_zoom, terrain_dir):
    result = []

    result.append(
        ShellCommandStep(['yum', '-y', 'install', 'vaisala-radarsw-gis'],
                         u"Install basemap dependencies"))

    basemap_installer_command = ["rsw-basemap-installer",
                                 "--gis-db-dump", gis_db_dump]

    if terrain_dir is None:
        basemap_installer_command += ['--skip-terrain']
    else:
        basemap_installer_command += ['--terrain-dir', terrain_dir]

    if max_zoom is not None:
        basemap_installer_command += ['--max-zoom', str(max_zoom)]

    result.append(ShellCommandStep(basemap_installer_command,
                                   "Install basemap server"))

    return result


def create_geoserver_site_config_steps(socket_server_address, geoserver_config_url):
    parsed = urlparse.urlparse(geoserver_config_url)
    result = [
        FunctionStep(lambda: wait_until_code(302, parsed.netloc, parsed.path,
                                             "GeoServer", timeout_secs=300),
                     "Wait for basemap GeoServer to start up")
    ]

    # GeoServer site setup
    if "localhost" in geoserver_config_url:
        result.append(
            ShellCommandStep(['rsw-basemap-site-setup',
                              '--socket-server', socket_server_address],
                             u"Configure radar sites for local GeoServer"))
    else:
        result.append(
            ShellCommandStep(["/usr/vaisala/radarsw/configuration/bin/configure-geoserver",
                              "get_centers", "-s", socket_server_address],
                             "Fetch radar centers from socket server"))
        result.append(
            ShellCommandStep(["/usr/vaisala/radarsw/configuration/bin/configure-geoserver",
                              "configure", "-url", geoserver_config_url],
                             "Configure remote GeoServer with correct radar centers"))

    result.append(
        ShellCommandStep(["service", "vaisala-radarsw-webapp", "restart"],
                         "Restart IRIS Focus after site configuration"))
    result.append(
        FunctionStep(lambda: wait_until_code(200, "localhost:34080", "/health", "IRIS Focus"),
                     "Wait for IRIS Focus to start up"))

    return result


def create_fresh_install_steps(args, socket_server):
    prerequisite_steps = [
        BASE_REPO_AVAILABLE_STEP,
        HOSTNAME_RESOLVABLE_STEP,
        FunctionStep(lambda: socket_server_version_check(socket_server),
                     u"Check for IRIS Socket Server version",
                     prerequisite=True),
        make_verify_executable_is_callable_step("rsw-rpm-radarsw-repo-create"),
    ]

    if not args.skip_geoserver_installation:
        prerequisite_steps.append(make_gis_dump_existence_check_step(args.gis_db_dump))

        if not args.skip_terrain:
            prerequisite_steps.append(make_terrain_dir_existence_check_step(args.terrain_dir))

    package_installation_steps = [
        BASE_REPO_SETUP_STEP,
        make_remove_postgres_92_packages_step(),
        ShellCommandStep(["rsw-rpm-radarsw-repo-create"], u"Install 'radarsw' Yum repo"),
        ShellCommandStep(install_packages_command(), u"Install RPM packages",
                         help_message=
"""Couldn't install packages. If you encountered a problem with Yum repos being
offline, try disabling all repos that have 0 packages available or when you
encounter an error such as 'Cannot find a valid baseurl for repo: <repo-id>'.

You can list all enabled repos with command
 - yum repolist enabled

You'll see the package count on the rightmost column. You can disable the repo
by calling
 - yum-config-manager --disable <repo-id>
where <repo-id> is the left-most column in the previous listing."""),
    ]
    steps = prerequisite_steps + package_installation_steps

    app_installation_steps = [
        OutputSuppressingShellCommandStep(
            ["rsw-postgresql-94-configure"], "Configure PostgreSQL database server",
            ['"/var/lib/pgsql/9.4/data" is missing or empty.',
             'Use "/usr/pgsql-9.4/bin/postgresql94-setup initdb" to initialize the database cluster.',
             'See %{_pkgdocdir}/README.rpm-dist for more information.',
             'MISSING              =>',
             '128MB                =>',
             "'< %m >'             =>"]),

        ShellCommandStep(["service", "postgresql-9.4", "start"], "Start up PostgreSQL database server"),
        ShellCommandStep(["rsw-db-tool", "create"], "Create database for IRIS Focus"),
        ShellCommandStep(configure_iris_host_command(socket_server), "Configure IRIS Socket Server host"),
        ShellCommandStep(["service", "vaisala-radarsw-webapp", "restart"],
                         "Restart IRIS Focus after socket server configuration"),
        FunctionStep(lambda: wait_until_code(200, "localhost:34080", "/health", "IRIS Focus"),
                     "Wait for IRIS Focus to start up after socket server configuration"),
        ShellCommandStep(["/usr/vaisala/radarsw/configuration/bin/configure-map",
                          "-u", args.wms_url, args.admin_user, args.admin_password],
                         "Configure IRIS Focus basemap WMS URL"),
        ShellCommandStep(["/usr/vaisala/radarsw/webapp-proxy/bin/generate-temp-ssl-cert"],
                         "Generate temporary self-signed SSL certificate"),
        ShellCommandStep(["service", "haproxy", "start"],
                         "Start up HTTP proxy, load balancer and SSL termination")
    ]
    steps = steps + app_installation_steps

    if not args.skip_geoserver_installation:
        steps += create_basemap_installation_steps(args.gis_db_dump, args.max_zoom, args.terrain_dir)

    if not args.skip_geoserver_site_configuration:
        steps += create_geoserver_site_config_steps(args.socket_server,
                                                    args.geoserver_config_url)

    steps.append(ShellCommandStep(["service", "monit", "start"],
                                  "Restart service monitoring daemon monit"))

    steps.append(BASE_REPO_CLEANUP_STEP)
    return steps


def create_upgrade_steps(args, currently_installed_version,
                         config_file, config_file_rpmsave):
    if currently_installed_version.startswith('1.'):
        return create_1_x_to_2_0_0_upgrade_steps(args, config_file, config_file_rpmsave)
    else:
        sys.exit(
            u"Upgrade of currently installed version of {0} not supported.".format(
            currently_installed_version))


def create_1_x_to_2_0_0_upgrade_steps(args, config_file, config_file_rpmsave):
    prerequisite_steps = [
        BASE_REPO_AVAILABLE_STEP,
        HOSTNAME_RESOLVABLE_STEP,
        FunctionStep(lambda: socket_server_version_check(args.socket_server),
                     u"Check for IRIS Socket Server version",
                     prerequisite=True),
        make_verify_executable_is_callable_step("rsw-postgresql-upgrade-92-to-94"),
        make_verify_executable_is_callable_step("rsw-rpm-radarsw-repo-create"),
        ShellCommandStep(["service", "vaisala-radarsw-webapp", "start"],
                         "Start IRIS Vision (if not running)", prerequisite=True),
        FunctionStep(lambda: wait_until_code(200, "localhost:34080", "/health", "IRIS Vision"),
                     "Wait for IRIS Vision to be available", prerequisite=True),
        FunctionStep(lambda: can_log_in_as_admin(args.admin_password),
                     "Ensure we can log in as admin into IRIS Vision", prerequisite=True),
    ]

    if not args.skip_geoserver_installation:
        prerequisite_steps.append(make_gis_dump_existence_check_step(args.gis_db_dump))

        if not args.skip_terrain:
            prerequisite_steps.append(make_terrain_dir_existence_check_step(args.terrain_dir))

    upgrade_steps = [
        BASE_REPO_SETUP_STEP,
        ShellCommandStep(["rsw-rpm-radarsw-repo-create"],
                          u"Install 'radarsw' Yum repo"),

        ShellCommandStep(["service", "monit", "stop"], "Stop service monitoring daemon monit"),
        ShellCommandStep(["service", "vaisala-radarsw-webapp", "stop"],
                         "Stop IRIS Vision before update"),
        ShellCommandStep(["service", "vboxvmservice@vaisala-radarsw-geoserver", "stop"],
                         u"Stop legacy basemap VM (virtual machine)"),
        ShellCommandStep(["systemctl", "disable", "vboxvmservice@vaisala-radarsw-geoserver"],
                         u"Disable automatic startup of legacy basemap VM"),

        # Upgrade PostgreSQL server from 9.2 to 9.4.
        ShellCommandStep(["rsw-postgresql-upgrade-92-to-94"],
                         u"Upgrade PostgreSQL from 9.2 to 9.4"),

        # yum install will also update, but don't do anything with the VM here.
        ShellCommandStep(install_packages_command(), "Update RPM packages"),

        ShellCommandStep(configure_iris_host_command(args.socket_server),
                         "Copy IRIS Socket Server host config"),
        # For the db-tool migrate-c5-data part, we need a number of properties set in the env.
        FunctionStep(lambda: set_env_for_c5_data_migration(config_file, config_file_rpmsave),
                     "Set env properties for database data Carbon Five to Liquibase migration tool"),
        ShellCommandStep(["/usr/bin/rsw-db-tool", "migrate-c5-data"],
                         "Migrate database data for IRIS Focus"),
        ShellCommandStep(["/usr/bin/rsw-db-tool", "migrate"],
                         "Migrate database for IRIS Focus"),
    ]

    finalization_steps = [
        ShellCommandStep(["service", "vaisala-radarsw-webapp", "start"],
                         "Restart IRIS Focus"),
        ShellCommandStep(["/usr/vaisala/radarsw/configuration/bin/configure-map",
                          "-u", args.wms_url, args.admin_user, args.admin_password],
                         "Configure IRIS Focus basemap WMS URL"),
        ShellCommandStep(["service", "monit", "start"],
                         "Restart service monitoring daemon monit"),
    ]

    result = prerequisite_steps + upgrade_steps

    if not args.skip_geoserver_installation:
        result += create_basemap_installation_steps(args.gis_db_dump, args.max_zoom, args.terrain_dir)

    if not args.skip_geoserver_site_configuration:
        result += create_geoserver_site_config_steps(args.socket_server,
                                                     args.geoserver_config_url)

    return result + finalization_steps + [BASE_REPO_CLEANUP_STEP]


def log_system_information():
    log("###")
    log("## Begin system information section")
    log("###")

    def no_check_output(*args, **kwargs):
        try:
            return subprocess.check_output(*args, **kwargs)
        except Exception, e:
            return str(e)

    log(u"hostname={0}".format(no_check_output(['hostname']).replace('\n', '')))
    log("rsw-installer.md5sum={0}".format(no_check_output(['md5sum', sys.argv[0]]).replace('\n', '').strip().split(' ')[0]))
    log("uptime={0}".format(no_check_output(['uptime']).replace('\n', '').strip()))
    log("df-h=\n--------\n{0}\n--------".format(no_check_output(['df', '-h'])))
    log("mount=\n--------\n{0}\n--------".format(no_check_output(['mount'])))
    log("free-ht=\n--------\n{0}\n--------".format(no_check_output(['free', '-ht'])))
    log("cat-cpuinfo=\n--------\n{0}\n--------".format(no_check_output(['cat', '/proc/cpuinfo'])))
    log("###")
    log("## End of system information section")
    log("###\n")


def get_centos_version():
    try:
        with open("/etc/centos-release") as f:
            return f.readline().strip()
    except IOError:
        return "Unknown OS"


def ensure_centos_version():
    """Check that CentOS version is correct."""
    version = get_centos_version()
    if "release 7.1." in version:
        # all OK
        return
    print("Your current operating system is:", version)
    print("This software should be installed on CentOS 7.1.")
    print("Are you sure you want to continue (y/n)?")
    inp = raw_input().strip().lower()
    if inp == 'y':
        # let's continue
        return
    sys.exit("Operating system version is incorrect")


def ensure_good_path_environment():
    """Adds the directory this executable resides in into PATH so that we can
    call rsw-rpm-radarsw-repo-create and various PostgreSQL upgrade-related scripts
    in the same dir."""
    try:
        subprocess.check_output(["rsw-rpm-radarsw-repo-create", "--test"])
        log("No changes needed for PATH environment variable.")
    except:
        installer_dir = os.path.dirname(os.path.realpath(__file__))
        os.environ['PATH'] = u"{}:{}".format(installer_dir, os.environ['PATH'])
        log(u"'{}' added to PATH environment variable.".format(installer_dir))


def main():
    parser = argparse.ArgumentParser('rsw-installer',
                                     formatter_class=argparse.ArgumentDefaultsHelpFormatter)
    subparsers = parser.add_subparsers(title="supported commands",
                                       dest='subparser_name')

    installer_parser = subparsers.add_parser('install', help='Installation of IRIS Focus.',
                                             formatter_class=argparse.ArgumentDefaultsHelpFormatter)
    req_group_install = installer_parser.add_argument_group("required arguments")
    req_group_install.add_argument("-s", dest="socket_server",
                                  help="IRIS socket server.", required=True,
                                  metavar="SOCKET SERVER HOST")
    installer_parser.add_argument("--admin-password", dest="admin_password", default="admin123", required=False)

    upgrade_parser = subparsers.add_parser('upgrade', help='Upgrade to IRIS Focus.',
                                           formatter_class=argparse.ArgumentDefaultsHelpFormatter)
    upgrade_parser.add_argument("-s", dest="socket_server",
                                help="IRIS socket server.", required=False,
                                metavar="SOCKET SERVER HOST")
    req_group_upgrade = upgrade_parser.add_argument_group("required arguments")
    req_group_upgrade.add_argument("--admin-password", dest="admin_password", required=True)

    def add_common_arguments(parser):
        group = parser.add_argument_group("common options")
        group.add_argument("-d", "--debug", dest="debug", action="store_true", help="Enable debug messages")

        group.add_argument("--dry-run", dest="dry_run", action="store_true",
                           default=False,
                           help="Don't do anything, just list the steps that would be done.")
        group.add_argument("-c", "--config-dir", dest="config_dir", default=
                            os.environ.get("VAISALA_RADARSW_CONFIG_DIR", '/etc/vaisala/radarsw/configuration'),
                            metavar="DIRECTORY")

        group.add_argument("--admin-user", dest="admin_user", default="admin")

        group.add_argument("--skip-geoserver-installation",
                           dest="skip_geoserver_installation",
                           default=False, action="store_true")
        group.add_argument("--gis-db-dump", dest="gis_db_dump", metavar="PATH",
                           # default="./gis_db_dump", # TODO: need a sane default
                           help="GIS database dump.")
        group.add_argument("--terrain-dir", dest="terrain_dir", metavar="DIRECTORY",
                           # default="./gis_db_dump", # TODO: need a sane default
                           help="Terrain data directory.")
        group.add_argument("--skip-terrain",
                           dest="skip_terrain",
                           default=False, action="store_true")

        group.add_argument("-g", "--geoserver-config-url", metavar="URL",
                            dest="geoserver_config_url", help="GeoServer configuration endpoint.",
                            default="http://localhost:34180/geoserver")
        group.add_argument("-w", "--wms", dest="wms_url", help="Basemap WMS address.",
                            default="/wms", metavar="URL")
        group.add_argument("--skip-geoserver-site-configuration",
                           dest="skip_geoserver_site_configuration",
                           default=False, action="store_true")
        group.add_argument("--max-zoom", dest="max_zoom", type=int,
                           help=argparse.SUPPRESS)

        group.add_argument("--online", dest="online", default=False, action="store_true",
                           help="Allow online CentOS base repository")
        group.add_argument("--offline", dest="offline", default=False, action="store_true",
                           help="Disable online CentOS base repository and require a " +
                           "file://-protocol CentOS base repository")

    add_common_arguments(installer_parser)
    add_common_arguments(upgrade_parser)

    args = parser.parse_args()

    if args.online == args.offline:
        parser.error("one of --online and --offline must be specified!")
    global REPO_STATE
    REPO_STATE = RepoState(args.online)

    if args.skip_terrain is False and args.terrain_dir is None:
        parser.error("--terrain-dir required if --skip-terrain is not specified")

    # Set up logging
    if not args.dry_run:
        log_file_path = os.path.join(os.environ['HOME'],
                                     u"rsw-installer-{0}.log".format(time.strftime("%Y%m%d-%H%M%S")))
        file_handler = logging.FileHandler(log_file_path)
        file_handler.setLevel(logging.DEBUG)
        _logger.addHandler(file_handler)

    log_system_information()

    stdout_handler = logging.StreamHandler(sys.stdout)
    if args.debug:
        stdout_handler.setLevel(logging.DEBUG)
        _logger.setLevel(logging.DEBUG)
    else:
        stdout_handler.setLevel(logging.INFO)
        _logger.setLevel(logging.INFO)
    _logger.addHandler(stdout_handler)
    if not args.dry_run:
        log(u"Logging to stdout and to '{0}'.".format(log_file_path))
    # End of logging setup

    if pwd.getpwuid(os.getuid()).pw_name != 'root':
        log("Installer must be run as root. Exiting.")
        sys.exit("This script must be run as root.")

    log("Current directory: {}".format(os.getcwd()))
    log("Started with command line {}".format(" ".join(sys.argv)))

    ensure_centos_version()
    ensure_good_path_environment()

    if "VAISALA_RADARSW_CONFIG_DIR" not in os.environ:
        os.environ["VAISALA_RADARSW_CONFIG_DIR"] = args.config_dir
    config_file = os.path.join(args.config_dir, 'vsoweb-override.ini')
    config_file_rpmsave = config_file + ".rpmsave"

    def log_installed_vaisala_radarsw_packages():
        try:
            log("Currently installed vaisala-radarsw packages:")
            log(
                subprocess.check_output(
                    "rpm -qa --qf '%{NAME} %{VERSION}\n' | grep vaisala-radarsw-",
                    shell=True))
            log()
        except:
            pass

    if not args.skip_geoserver_installation:
        if args.gis_db_dump is None:
            parser.error("If --skip-geoserver-installation is not specified, --gis-db-dump must be set.")

    if args.subparser_name == 'upgrade':
        # TODO: make sure we have old version installed
        currently_installed_version = resolve_webapp_version()
        dbg("Currently installed version (blank for none): \"{0}\"".format(currently_installed_version))

        if not currently_installed_version.startswith("1."):
            log_installed_vaisala_radarsw_packages()
            parser.error("No current installed IRIS Vision found.")

        args.socket_server = resolve_socket_server_address(
            args.socket_server, upgrade_parser, config_file, config_file_rpmsave)
        steps = create_upgrade_steps(args, currently_installed_version, config_file, config_file_rpmsave)

    elif args.subparser_name == 'install':
        currently_installed_version = resolve_webapp_version()
        if currently_installed_version.startswith("1."):
            log_installed_vaisala_radarsw_packages()
            parser.error("IRIS Vision already installed. Use 'upgrade' to upgrade.")
        elif currently_installed_version.startswith("2."):
            log_installed_vaisala_radarsw_packages()
            print("Some IRIS Focus packages are already installed.")
            print("Are you sure you want to continue (y/n)?")
            inp = raw_input().strip().lower()
            if inp != 'y':
                sys.exit("IRIS Focus already installed.")
        steps = create_fresh_install_steps(args, args.socket_server)

    dbg(u"Using {0} as the IRIS socket server address.".format(args.socket_server))
    dbg(u"Using {0} as the basemap WMS URL.".format(args.wms_url))
    dbg(u"Using {0} as the GeoServer configuration URL.".format(args.geoserver_config_url))

    for idx, step in enumerate(steps):
        if idx > 0:
            log()

        if step.prerequisite is True:
            log(u"## Radar Software Installer Prerequisite Step {0}/{1}".format(idx + 1, len(steps)))
        else:
            log(u"## Radar Software Installer Step {0}/{1}".format(idx + 1, len(steps)))

        while True:
            try:
                if args.dry_run:
                    step.dry_run()
                else:
                    step()
                break
            except StepFailed, f:
                try:
                    log(u"Step failed: {0}".format(str(f)))
                    if step.help_message is not None:
                        log()
                        log(step.help_message)
                        log()
                    skipretryfail = skip_retry_fail()
                    if skipretryfail == 'skip':
                        break
                    elif skipretryfail == 'retry':
                        pass
                    elif skipretryfail == 'fail':
                        raise f
                except KeyboardInterrupt, kbi:
                    _logger.error(traceback.format_exc())
                    raise kbi

    if not args.dry_run:
        log(u"You can find the installation log at '{0}'.".format(log_file_path))


if __name__ == '__main__':
    try:
        main()
    finally:
        try:
            _base_repo_cleanup()
        except:
            pass
