#!/usr/bin/env python 
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
# 
#   http://www.apache.org/licenses/LICENSE-2.0
# 
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
# -*- indent-tabs-mode: nil; tab-width:4 -*-
# vim:set tabstop=4 expandtab:
'''
gpssh-exkeys -- exchange ssh public keys among friends

Usage: gpssh-exkeys [--version] [-?v][-p] 
                    { -f hostfile |
                      -h host ... |
                      -e hostfile -x hostfile }

             --version     : print version information
             -?            : print this help screen
             -v            : verbose mode
             -p password   : the password used to connect to hosts
             -h host       : the new host to connect to (multiple -h is okay)
             -f hostfile   : a file listing all new hosts to connect to
             -e hostfile   : a file listing all existing hosts for expansion
             -x hostfile   : a file listing all new hosts for expansion
             
    Each line in a hostfile is expected to contain a single host name.  Blank
    lines and comment lines (beginning with #) are ignored.  The name of the
    local host (as provided by hostname) is included automatically and need not 
    be specified unless it is the only host to process.  During cluster expansion, 
    the local host is always considered an existing host and should not be specified
    in the "new host" list.  Duplicate host names in either the new host list (-h, 
    -f, -x options) or the existing host list (-e option) are ignored. The same host 
    name cannot appear in the both the new and existing host lists. Host names 
    including a user name or port (username@hostname:port) are not accepted.
'''

from __future__ import with_statement 
import os, sys

progname = os.path.split(sys.argv[0])[-1]

if sys.version_info < (2, 5, 0):
    sys.exit(
'''Error: %s is supported on Python versions 2.5 or greater
Please upgrade python installed on this machine.''' % progname)

#disable deprecationwarnings
import warnings
warnings.simplefilter('ignore', DeprecationWarning)

sys.path.append(sys.path[0] + '/lib')
try:
    import getopt, time, getpass, logging
    import tempfile, filecmp
    import array, socket, subprocess
    import pexpect
    from gppylib.commands import unix
    from gppylib.util import ssh_utils
    from gppylib.gpparseopts import OptParser
    from gppylib.gpcoverage import GpCoverage
except ImportError, e:
    sys.exit('Error: unable to import module: ' + str(e))


#
# all the command line options
#

class Global:
    script_name = os.path.split(__file__)[-1]
    opt = {}
    opt['-v'] = False
    opt['-h'] = []
    opt['-f'] = False
    opt['-x'] = False           # new hosts for expansion
    opt['-e'] = False           # existing hosts file for expansion
    passwd = []
    # ssh commands don't respect $HOME; they always use the home
    # directory supplied in /etc/passwd so sshd can find the same
    # directory.
    homeDir = os.path.expanduser("~" + unix.getUserName())
    authorized_keys_fname = '%s/.ssh/authorized_keys' % homeDir
    known_hosts_fname = '%s/.ssh/known_hosts' % homeDir
    id_rsa_fname = '%s/.ssh/id_rsa' % homeDir
    id_rsa_pub_fname = id_rsa_fname + '.pub'
    allHosts = []                   # all hosts, new and existing, to be processed
    newHosts = []                   # new hosts for initial or expansion processing
    existingHosts = []              # existing hosts for expansion processing

GV = Global()

################
def usage(exitarg):
    parser = OptParser()
    try:
        parser.print_help()
    except:
        print __doc__
    sys.exit(0)

#############
def print_version():
    print '%s version $Revision$' % GV.script_name
    sys.exit(0)


class Host: 
    def __init__(self, host, localhost=False):
        self.m_host = host
        self.m_popen = None
        self.m_popen_cmd = ''
        self.m_remoteID = None
        self.m_isLocalhost = localhost
        self.m_inetAddrs = None
        self.m_inet6Addrs = None
        
    def __repr__(self):
        return ('(%s, { "popen" : %s, "remoteId" : %s, "popen_cmd" : "%s" })' 
                % (self.m_host, (True if self.m_popen else False), self.m_remoteID, self.m_popen_cmd))

    def host(self): return self.m_host
    def remoteID(self): return self.m_remoteID
    def popen_cmd(self): return self.m_popen_cmd;
    def isPclosed(self): return self.m_popen == None;
    
    
    def getAddrs(self):
        '''
        Gets the INET and INET6 addresses for this host.
        '''
        if (self.m_inetAddrs == None) and (self.m_inet6Addrs == None):
            self.m_inetAddrs = []
            self.m_inet6Addrs = []
            try:
                hostAddrs = socket.getaddrinfo(self.m_host, 0, socket.AF_UNSPEC, socket.SOCK_STREAM, socket.IPPROTO_TCP, 0)
                
                if self.m_isLocalhost:
                    try:
                        hostAddrs.extend(socket.getaddrinfo('localhost', 0, socket.AF_UNSPEC, socket.SOCK_STREAM, socket.IPPROTO_TCP, 0))
                    except:
                        pass
                
                for (family, socktype, proto, canonname, sockaddr) in hostAddrs:
                    if family == socket.AF_INET:
                        (addr, port) = sockaddr
                        self.m_inetAddrs.append(addr)
                    elif family == socket.AF_INET6:
                        (addr, port, flowinfo, scopeid) = sockaddr
                        self.m_inet6Addrs.append(addr)
            except socket.gaierror:
                pass
            self.m_inetAddrs = tuple(self.m_inetAddrs)
            self.m_inet6Addrs = tuple(self.m_inet6Addrs)
            
        return (self.m_inetAddrs, self.m_inet6Addrs)
    
    
    def isSameHost(self, host):
        '''
        Compares <host> with this host by published address
        '''
        (thisInetAddrs, thisInet6Addrs) = self.getAddrs()
        (thatInetAddrs, thatInet6Addrs) = host.getAddrs()
        
        for addr in thisInetAddrs:
            if addr in thatInetAddrs:
                return True
        for addr in thisInet6Addrs:
            if addr in thatInet6Addrs:
                return True
            
        return False


    def password_ssh(self, host, cmd, user, password, timeout=5):
        """SSH to a host using the supplied password and executes a command."""
        fname = tempfile.mktemp()
        fout = open(fname, 'w')

        options = '-q -oPasswordAuthentication=yes -oStrictHostKeyChecking=no -oUserKnownHostsFile=/dev/null -oPubkeyAuthentication=no'
        ssh_cmd = 'ssh %s@%s %s "%s"' % (user, host, options, cmd)

        try:
            child = pexpect.spawn(ssh_cmd, timeout=timeout)
            child.expect(['password: '])
            child.sendline(password)
            child.logfile = fout
            child.expect(pexpect.EOF)
            child.close()
            fout.close()
        except:
            pass

        fin = open(fname, 'r')
        stdout = fin.read()
        fin.close()

        return stdout, child.exitstatus


    def sendLocalID(self, ID, passwd, tempDir):
        '''
        Send local ID to remote over SSH, and append to authorized_key.
        If <tempDir> is specified, the authorized_keys, known_hosts, and
        id_rsa.pub files are obtained from the target host.  These files
        are placed in <tempDir>/<self.m_host>
        '''
        cur_user = getpass.getuser()
        password_less = True
        cmd = 'true'
        rc = os.system("ssh -oStrictHostKeyChecking=no -oPasswordAuthentication=no -oPubkeyAuthentication=yes %s \"%s\"" % (self.m_host, cmd))
        if rc != 0:
            password_less =  False
            for pwd in passwd:
                output, rc = self.password_ssh(self.m_host, cmd, cur_user, pwd)
                if rc == 0:
                    cur_password = pwd
                    break
            while rc != 0:
                print >> sys.stderr, '  ***'
                pwd = getpass.getpass('  *** Enter password for %s: ' % self.m_host, sys.stderr)
                if pwd:
                    output, rc = self.password_ssh(self.m_host, cmd, cur_user, pwd)
                if rc == 0:
                    cur_password = pwd
                    passwd.append(pwd)

        # Create .ssh directory and ensure content meets permission requirements
        # for password-less SSH
        #
        # note: we touch .ssh/iddummy.pub just before the chmod operations to
        # ensure the wildcard matches at least one file.
        cmd = ('mkdir -p .ssh; ' +
               'chmod 0700 .ssh; ' +
               'touch .ssh/authorized_keys; ' +
               'touch .ssh/known_hosts; ' +
               'touch .ssh/config; ' +
               'touch .ssh/iddummy.pub; ' +
               'chmod 0600 .ssh/auth* .ssh/id*; ' +
               'chmod 0644 .ssh/id*.pub .ssh/config')
        if GV.opt['-v']: print '[INFO %s]: %s' % (self.m_host, cmd)
        if password_less:
            rc = os.system("ssh -oStrictHostKeyChecking=no -oPasswordAuthentication=no -oPubkeyAuthentication=yes %s \"%s\"" % (self.m_host, cmd))
        else:
            output, rc = self.password_ssh(self.m_host, cmd, cur_user, cur_password)

        if rc != 0:
            print "Set permission for ssh configuration files on host %s failed, exit." % self.m_host
            return False

        # If tempDir is specified, obtain a copy of the ssh
        # files that should be preserved for existing hosts.
        if tempDir:
            cmd = 'cd .ssh && tar cf %s.tar authorized_keys known_hosts id_rsa.pub' % self.m_host
            if GV.opt['-v']: print '[INFO %s]: %s' % (self.m_host, cmd)
            if password_less:
                rc = os.system("ssh -oStrictHostKeyChecking=no -oPasswordAuthentication=no -oPubkeyAuthentication=yes %s \"%s\"" % (self.m_host, cmd))
            else:
                output, rc = self.password_ssh(self.m_host, cmd, cur_user, cur_password)
            if rc != 0:
                print "Backup ssh configuration files on host %s failed, exit." % self.m_host
                return False

        # Append local ID to authorized_keys
        if not password_less:
            cmd = 'echo \"%s\" >> .ssh/authorized_keys && echo ok ok ok; chmod 0600 .ssh/authorized_keys;' % ID
            if GV.opt['-v']: print '[INFO %s]: %s' % (self.m_host, cmd)
            output, rc = self.password_ssh(self.m_host, cmd, cur_user, cur_password)
            if rc != 0:
                print "Append local ID to authorized_keys on %s failed, exit." % self.m_host
                return False

        if tempDir:
            rc = os.system("scp -q -oStrictHostKeyChecking=no -oPasswordAuthentication=no -oPubkeyAuthentication=yes %s:%s/.ssh/%s.tar  %s/" % (self.m_host, os.environ['HOME'], self.m_host, tempDir))
            if rc != 0:
                print "Copy backup file %s.tar from remote host %s failed, exit." % (self.m_host, self.m_host)
                return False

        return True


    def popen(self, cmd):
        'Run a command and save popen handle in this Host instance.'
        if self.m_popen: 
            self.m_popen.close()
            self.m_popen = None
        if GV.opt['-v']: print '[INFO %s]: %s' % (self.m_host, cmd)
        self.m_popen = os.popen(cmd)
        self.m_popen_cmd = cmd
        return self.m_popen

    def pclose(self):
        'Close the popen handle'
        if not self.m_popen: return (False, None)
        content = self.m_popen.read()
        ok = not self.m_popen.close()
        self.m_popen = None
        return (ok, content)

    def setRemoteID(self, ID):
        'Save the remote ID'
        self.m_remoteID = ID


def parseCommandLine():
    global opt
    try:
        (options, args) = getopt.getopt(sys.argv[1:], '?vh:p:f:x:e:', ['version'])
    except Exception, e:
        usage('[ERROR] ' + str(e))

    for (switch, val) in options:
        if (switch == '-?'):          usage(0)
        elif (switch == '-v'):        GV.opt[switch] = True
        elif (switch[1] in ['f', 'x', 'e']):      GV.opt[switch] = val
        elif (switch == '-h'):        GV.opt[switch].append(val)
        elif (switch == '-p'):        GV.passwd.append(val)
        elif (switch == '--version'): print_version()

    if not (len(GV.opt['-h']) or GV.opt['-f'] or GV.opt['-x'] or GV.opt['-e']):
        usage('[ERROR] please specify at least one of the -h or -f args or both the -x and -e args')
    elif len(GV.opt['-h']) or GV.opt['-f']:
        if (GV.opt['-x'] or GV.opt['-e']):
            usage('[ERROR] an -h or -f arg may not be specified with the -x and -e args')
        elif len(GV.opt['-h']) and GV.opt['-f']:
            usage('[ERROR] please specify either an -h or -f arg, but not both')
    elif not (GV.opt['-x'] and GV.opt['-e']):
        usage('[ERROR] the -x and -e args must be specified together')


###  collect hosts for HostList
#
#
def collectHosts(hostlist, hostfile):
    '''
    Adds hosts from hostfile to hostlist
    '''
    try:
        hostlist.parseFile(hostfile)
    except ssh_utils.HostNameError:
        print >> sys.stderr, '[ERROR] host name %s in file %s is not supported' % (str(sys.exc_info()[1]), hostfile)
        sys.exit(1)
    if not hostlist.get():
        usage('[ERROR] no valid hosts specified in file %s' % hostlist)


###  create local id_rsa if not already available
#
#    Returns the content of if_rsa.pub for the generated or existing key pair.
def createLocalID():
    if os.path.exists(GV.id_rsa_fname):
        print '  ... %s file exists ... key generation skipped' % GV.id_rsa_fname
    else :
        errfile = os.path.join(tempDir, "keygen.err")
        cmd = 'ssh-keygen -t rsa -N \"\" -f %s < /dev/null >/dev/null 2>%s' % (GV.id_rsa_fname, errfile)
        if GV.opt['-v']: print '[INFO] executing', cmd
        rc = os.system(cmd)
        if rc:
            print >> sys.stderr, '[ERROR] ssl-keygen failed:'
            for line in open(errfile):
                print >> sys.stderr, '    ' + line.rstrip()
            sys.exit(rc)

    f = None; 
    try:
        try: 
            f = open(GV.id_rsa_pub_fname, 'r'); 
            return f.readline().strip()
        except IOError:
            sys.exit('[ERROR] ssh-keygen failed - unable to read the generated file ' + GV.id_rsa_pub_fname)
    finally:
        if f: f.close()


### Append the id_rsa.pub value provided to authorized_keys
def authorizeLocalID(localID):
    # Check the current authorized_keys file for the localID
    f = None
    try:
        f = open(GV.authorized_keys_fname, 'a+')
        for line in f:
            if line.strip() == localID:
                # The localID is already in authorizedKeys; no need to add
                return
        if GV.opt['-v']: print '[INFO] appending localID to authorized_keys'
        f.write(localID)
        f.write('\n')
    finally:
        if f: f.close()


def testAccess(hostname):
    '''
    Ensure the proper password-less access to the remote host.
    Using ssh here also allows discovery of remote host keys *not*
    reported by ssh-keyscan.
    '''
    errfile = os.path.join(tempDir, 'sshcheck.err')
    cmd = 'ssh -o "BatchMode=yes" -o "StrictHostKeyChecking=no" %s true 2>%s' % (hostname, errfile)
    if GV.opt['-v']: print '[INFO %s]: %s' % (hostname, cmd)
    rc = os.system(cmd)
    if rc != 0:
        print >> sys.stderr, '[ERROR %s] authentication check failed:' % hostname
        with open(errfile) as efile:
            for line in efile:
                print >> sys.stderr, '    ', line.rstrip()
        return False
    
    return True


def addRemoteID(tab, line):
    IDKey = line.strip().split()
    if not (len(IDKey) == 3 and line[0] != '#'): return False
    tab[IDKey[2]] = line
    return True


def readAuthorizedKeys(tab=None, keysFile=None):
    if not keysFile: keysFile = GV.authorized_keys_fname
    f = None
    if not tab: tab = {}
    try:
        f = open(keysFile, 'r')
        for line in f: addRemoteID(tab, line)
    finally:
        if f: f.close()
    return tab


def writeAuthorizedKeys(tab, keysFile=None):
    if not keysFile: keysFile = GV.authorized_keys_fname
    f = None
    try: 
        f = open(keysFile, 'w')
        for IDKey in tab: f.write(tab[IDKey])
    finally:
        if f: f.close()

def addKnownHost(tab, line):
    key = line.strip().split()
    if not (len(key) == 3 and line[0] != '#'): return False
    tab[key[0]] = line
    return True

def readKnownHosts(tab=None, hostsFile=None):
    if not hostsFile: hostsFile = GV.known_hosts_fname
    f = None
    if not tab: tab = {}
    try:
        f = open(hostsFile, 'r')
        for line in f: addKnownHost(tab, line)
    finally:
        if f: f.close()
    return tab

def writeKnownHosts(tab, hostsFile=None):
    if not hostsFile: hostsFile = GV.known_hosts_fname
    f = None
    try: 
        f = open(hostsFile, 'w')
        for key in tab: f.write(tab[key])
    finally:
        if f: f.close()

def addHost(hostname, hostlist, localhost=False):
    '''
    Adds a Host(hostname) entry to hostlist if not a "localhost" and not already in the
    list (by name).  Returns True if hostname was added; False otherwise.
    '''
    if (hostname + '.').startswith("localhost.") or (hostname + '.').startswith("localhost6"):
        return False
    for host in hostlist:
        if host.host() == hostname:
            return False
    hostlist.append(Host(hostname, localhost))
    return True



tempDir = None

coverage = GpCoverage()
coverage.start()

try:
    if os.environ.has_key('_GPDEBUG'):
        debugLog = logging.StreamHandler()
        debugLog.setLevel(logging.DEBUG)
        debugLog.setFormatter(logging.Formatter("%(asctime)s - %(name)s - %(module)s:%(funcName)s:%(lineno)d - %(levelname)s - %(message)s"))
    else:
        nullFile = logging.FileHandler('/dev/null')

    parseCommandLine()
    
    # Assemble a list of names used by the current host.  SSH is sensitive to both name
    # and address so recognizing each name can prevent an SSH authenticitiy challenge.
    #
    # We start out with the names presented by gethostname and getfqdn (which may be the
    # same or localhost) and add to this list using gethostbyaddr to discover possible
    # aliases.
    localhosts = []
    for hostname in (socket.gethostname(), socket.getfqdn()):
        if addHost(hostname, localhosts, True):
            (primary, aliases, ipaddrs) = socket.gethostbyaddr(hostname)
            addHost(primary, localhosts, True)
            for alias in aliases:
                addHost(alias, localhosts, True)
    localhosts = tuple(localhosts)
    
    # hostlist is the collection of "new" hosts; it is composed of hosts
    # identified by the -h or -f options for initial exchange processing
    # or by the -x option for expansion processing.  (Only one of the -h,
    # -f, or -x options is expected to have values.)
    hostlist = ssh_utils.HostList()
    
    if len(GV.opt['-h']):
        for h in GV.opt['-h']:
            try:
                hostlist.add(h)
            except ssh_utils.HostNameError:
                print >> sys.stderr, '[ERROR] host name %s is not supported' % str(sys.exc_info()[1])
                sys.exit(1)
        if not hostlist.get():
            usage('[ERROR] no valid hosts specified in -h arguments')

    if GV.opt['-f']: collectHosts(hostlist, GV.opt['-f'])
    if GV.opt['-x']: collectHosts(hostlist, GV.opt['-x'])
    # Add all the fqdn and hostname if not already existing
    hostlist.addHostNameAlternatives()

    # Check the new host list for (1) the current (local) host and (2) duplicate
    # host identifiers.  If the local host appears in the new list, leave it for
    # the time being ... it is removed later.
    localhostInNew = False
    for host in hostlist.get():
        host = Host(host)
        for localhost in localhosts:
            if localhost.host() == host.host():
                localhostInNew = True
                host = localhost
                continue
        for h in GV.newHosts:
            if h.host() == host.host():
                break
        else:
            GV.newHosts.append(host)
            
    if not GV.newHosts:
        print >> sys.stderr, '[ERROR] no valid new hosts specified; at least one new host must be specified for key exchange'
        sys.exit(1)
    
    GV.allHosts.extend( GV.newHosts )

    # hostlist is now used for the collection of existing hosts.
    # (The existing hosts list will exist iff the -x option is used
    # for new hosts.)
    localhostInOld = False
    hostlist = ssh_utils.HostList()
    if GV.opt['-e']: 
        collectHosts(hostlist, GV.opt['-e'])
        # Add the fqdn and short names if not already added
        hostlist.addHostNameAlternatives()
        
        for host in hostlist.get():
            host = Host(host)
            for localhost in localhosts:
                if localhost.host() == host.host():
                    localhostInOld = True
                    host = localhost
                    continue
            for h in GV.existingHosts:
                if h.host() == host.host():
                    break
            else:
                GV.existingHosts.append(host)
                
        if not GV.existingHosts:
            print >> sys.stderr, '[ERROR] no valid existing hosts specified; at least one existing host must be specified for expansion'
            sys.exit(1)

        GV.allHosts.extend( GV.existingHosts )
        
        # Ensure there's no overlap between the new and existing hosts
        haveError = False
        for existingHost in GV.existingHosts:
            for newHost in GV.newHosts:
                if existingHost.host() == newHost.host():
                    print >> sys.stderr, '[ERROR] new host \"%s\" is the same as existing host \"%s\"' % (newHost.host(), existingHost.host())
                    haveError = True
                    break
        if haveError:
            sys.exit(1)
            
    # Ensure the local host is in the "proper" host list -- old for expansion, new otherwise
    if GV.opt['-e']:
        if localhostInOld:
            # Current host implicit in old list; remove explicit reference
            for localhost in localhosts:
                if localhost in GV.existingHosts:
                    GV.existingHosts.remove(localhost)
                if localhost in GV.allHosts:
                    GV.allHosts.remove(localhost)
    else:
        if localhostInNew:
            # Current host implicit in new list; remove explicit reference
            for localhost in localhosts:
                if localhost in GV.newHosts:
                    GV.newHosts.remove(localhost)
                if localhost in GV.allHosts:
                    GV.allHosts.remove(localhost)

    # Allocate a temporary directory; if KEEPTEMP is set, allocate the
    # directory in the user's home directory, otherwise use a system temp.
    if os.environ.has_key('KEEPTEMP'):
        tempDir = tempfile.mkdtemp('.tmp', 'gp_', os.path.expanduser('~'))
    else:
        tempDir = tempfile.mkdtemp()
    if GV.opt['-v'] or os.environ.has_key('KEEPTEMP'): 
        print '[INFO] tempDir=%s' % tempDir
    
    discovered_authorized_keys_file = os.path.join(tempDir, 'authorized_keys')

    ######################
    #  step 1
    #
    #    Creates an SSH id_rsa key pair for for the current user if not already available
    #    and appends the id_rsa.pub key to the local authorized_keys file.
    #
    print '[STEP 1 of 5] create local ID and authorize on local host'
    localID = createLocalID()
    authorizeLocalID(localID)
    
    # Ensure the local host's .ssh directory is prepared for password-less SSH login
    #
    # note: we touch .ssh/iddummy.pub just before the chmod operations to
    # ensure the wildcard matches at least one file.
    cmd = ('cd ' + GV.homeDir + '; ' +
           'chmod 0700 .ssh; ' +
           'touch .ssh/authorized_keys; ' +
           'touch .ssh/known_hosts; ' +
           'touch .ssh/config; ' +
           'touch .ssh/iddummy.pub; ' +
           'chmod 0600 .ssh/auth* .ssh/id*; ' +
           'chmod 0644 .ssh/id*.pub .ssh/config')
    if GV.opt['-v']: print '[INFO]: %s' % cmd
    os.system(cmd)
    
    # Ensure the host key(s) for the local host are in known_hosts.  Using ssh-keyscan
    # takes care of part of it; testAccess takes care of the rest.
    errfile = os.path.join(tempDir, "keyscan.err")
    for host in localhosts:
        cmd = 'ssh-keyscan -t rsa %s >> %s 2>%s' % (host.host(), GV.known_hosts_fname, errfile)
        if GV.opt['-v']: print '[INFO]', cmd
        rc = os.system(cmd)
        if rc != 0:
            print >> sys.stderr, ('[WARNING] error %s obtaining RSA host key(s) for local host %s'
                                  % (rc, host))
            for line in open(errfile):
                print >> sys.stderr, '    ' + line.rstrip()
        os.remove(errfile)
        # Test SSH access to local host to ensure proper inbound access and complete
        # known_hosts file.
        if not testAccess(host.host()):
            print >> sys.stderr, "[ERROR] cannot establish ssh access into the local host"
            sys.exit(1)

    ######################
    #  step 2
    #
    #    Interogate each host for its host key and add to the known_hosts file.
    #
    #    ssh-keyscan fails when supplied a non-existent host name so each host
    #    is polled separately.  Also, ssh-keyscan may not report all "hostname"
    #    information actually used by ssh; the first ssh-based contact will
    #    report a warning and update the known_hosts file if the key exists
    #    but the hostname is not as expected.
    #
    print; print '[STEP 2 of 5] keyscan all hosts and update known_hosts file'
    badHosts = []
    errfile = os.path.join(tempDir, "keyscan.err")
    for h in GV.allHosts:
        cmd = 'ssh-keyscan -t rsa %s >> %s 2>%s' % (h.host(), GV.known_hosts_fname, errfile)
        if GV.opt['-v']: print '[INFO]', cmd
        rc = os.system(cmd)
        if rc != 0:
            # If ssh-keyscan failed, it's typically because the host doesn't exist;
            # remove the host from further processing and inform the user
            print >> sys.stderr, ('[ERROR] error %s obtaining RSA host key for %s host %s' 
                                  % (rc,
                                     'existing' if h in GV.existingHosts else 'new',
                                     h.host()))
            for line in open(errfile):
                print >> sys.stderr, '    ' + line.rstrip()
            badHosts.append(h)
            GV.allHosts.remove(h)
            if h in GV.existingHosts: GV.existingHosts.remove(h)
            if h in GV.newHosts: GV.newHosts.remove(h)
    else:
        if len(badHosts):
            sys.exit('[ERROR] cannot process one or more hosts')

    ######################
    #  step 3
    #
    #    Temporarily append the localID to the authorized_keys file of 
    #    each host to allow password-less SSH.  This is a temporary measure --
    #    the authorized_keys file on each host is replaced in a later step.
    #
    #    This step also obtains a copy of any existing authorized_keys, 
    #    known_hosts, and id_rsa.pub files for existing hosts so they
    #    may be updated rather than replaced (as is done for new hosts).
    #
    #    The id_rsa.pub file from any existing host is collected for
    #    addition to this host's authorized_keys file and subsequent
    #    sharing with all hosts.
    #
    #    The last step for each host is ensuring that password-less access
    #    from the current user is enabled.  This is done using SSH rather
    #    than pexpect to ensure that normal SSH processing is possible.
    #
    print; print '[STEP 3 of 5] authorize current user on remote hosts'  # serial
    errmsg = None
    newKeys = None
    try:
        for h in GV.allHosts:
            print '  ... send to', h.host()
            isExistingHost = ( h in GV.existingHosts )
            send_local_id = False
            try:
                send_local_id = h.sendLocalID(localID, GV.passwd, tempDir if isExistingHost else None)
                #print "Send local id is: %s" % send_local_id
            except socket.error, e:
                errmsg = '[ERROR %s] %s' % (h.host(), e)
                print >> sys.stderr, errmsg 
            if not send_local_id: 
                errmsg = '[ERROR %s] skipping key exchange for %s' % (h.host(), h.host())
                print >> sys.stderr, errmsg
                errmsg = '[ERROR %s] unable to authorize current user' % h.host()
                print >> sys.stderr, errmsg
            else:
                if isExistingHost:
                    # Now extract the .ssh files from the tarball into the
                    # host-specific directory
                    tarfileName = os.path.join(tempDir, '%s.tar' % h.host())
                    hostDir = os.path.join(tempDir, h.host())
                    os.mkdir(hostDir)
                    cmd = 'cd %s && tar xf %s' % (hostDir, tarfileName)
                    if GV.opt['-v']: print '[INFO %s]: %s' % (h.host(), cmd)
                    tarproc = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
                    (tarout, tarerr) = tarproc.communicate()
                    if tarproc.returncode != 0:
                        print >> sys.stderr, '[WARNING %s] cannot extract SSH files;' % h.host()
                        for line in tarerr.splitlines():
                            print >> sys.stderr, '    ', line
                        print >> sys.stderr, '    One or more existing authentication files may be replaced on %s' % h.host()

                    hostId = os.path.join(hostDir, 'id_rsa.pub')
                    if os.path.exists(hostId) and not filecmp.cmp(GV.id_rsa_pub_fname, hostId):
                        if not newKeys:
                            newKeys = open(discovered_authorized_keys_file, 'w')
                        print '  ...... appending %s ID to authorized_keys' % h.host()
                        with open(hostId) as hostPub:
                            for line in hostPub:
                                newKeys.write(line)
                            newKeys.flush()
                
                # Ensure the proper password-less access to the remote host.
                if not testAccess(h.host()):
                    errmsg = '*'        # message already issued

            if errmsg: sys.exit(1)
    finally:
        if newKeys:
            newKeys.close()


    ######################
    #  step 4
    #
    #    At this point, 
    #        (1) the local known_hosts file has at least one 
    #            host key for each new and existing host.
    #        (2) the local authorized_keys file has an entry
    #            for the current user on the local system AND
    #            the public key from the current user on every
    #            existing host.
    #        (3) a copy of any existing authorized_keys, known_hosts, 
    #            and id_rsa.pub file from each existing host file, 
    #            exists in the <tempDir>/<host> directory.
    #
    #    Determine SSH authentication file content for each host.
    #    For new hosts, the authorized_keys, known_hosts, and 
    #    id_rsa{,.pub} files are copied from this host.  For
    #    existing hosts, the existing authorized_keys and known_hosts
    #    files from the existing host is merged with the files from
    #    this nost
    #
    print; print '[STEP 4 of 5] determine common authentication file content'

    # eliminate duplicates in known_hosts file
    # TODO: improve handling of hosts with multiple identifiers
    try:
        tab = readKnownHosts()
        writeKnownHosts(tab)
    except IOError:
        sys.exit('[ERROR] cannot read/write known_hosts file')

    # eliminate duploicates in authorized_keys file
    # TODO: improve handing of keys with optional elements
    try:
        tab = readAuthorizedKeys()
        # Now add any discovered user keys to the local authorized_keys file
        if os.path.exists(discovered_authorized_keys_file):
            print '  ... merging discovered remote IDs into local authorized_keys'
            tab = readAuthorizedKeys(tab, discovered_authorized_keys_file)
    except IOError: 
        sys.exit('[ERROR] cannot read authorized_keys file')

    try:
        writeAuthorizedKeys(tab)
    except IOError:
        sys.exit("[ERROR] unable to write authorized_keys file")


    ######################
    #  step 5
    #
    #    Set or update the authentication files on each remote host.
    #    For each new host, copy (and replace) the authorized_keys,
    #    known_hosts, and id_rsa{.,pub} files.  For existing hosts,
    #    merge the common authorized_keys and known_hosts content
    #    into the local copy of the remote host's files and replace
    #    the existing host's versions.
    #
    print; print '[STEP 5 of 5] copy authentication files to all remote hosts'
    errmsg = None
    
    try:

        # MPP-13617
        def canonicalize(s):
            if ':' not in s: return s
            return '\[' + s + '\]'

        for h in GV.newHosts:

            cmd = ('scp -q -o "BatchMode yes" -o "NumberOfPasswordPrompts 0" ' + 
                   '%s %s %s %s %s:.ssh/ 2>&1' 
                   % (GV.authorized_keys_fname, 
                      GV.known_hosts_fname, 
                      GV.id_rsa_fname, 
                      GV.id_rsa_pub_fname, 
                      canonicalize( h.host() )))
            h.popen(cmd)
    
        if len(GV.existingHosts):
            localAuthKeys = readAuthorizedKeys()
            localKnownHosts = readKnownHosts()
            
            for h in GV.existingHosts:
                
                remoteAuthKeysFile = os.path.join(tempDir, h.host(), 'authorized_keys')
                if os.path.exists(remoteAuthKeysFile) and os.path.getsize(remoteAuthKeysFile):
                    if GV.opt['-v']: print '  ... merging authorized_keys for %s' % h.host()
                    remoteAuthKeys = readAuthorizedKeys(localAuthKeys.copy(), remoteAuthKeysFile)
                    writeAuthorizedKeys(remoteAuthKeys, remoteAuthKeysFile)
                else:
                    remoteAuthKeysFile = GV.authorized_keys_fname
                
                remoteKnownHostsFile = os.path.join(tempDir, h.host(), 'known_hosts')
                if os.path.exists(remoteKnownHostsFile) and os.path.getsize(remoteKnownHostsFile):
                    if GV.opt['-v']: print '  ... merging known_hosts for %s' % h.host()
                    remoteKnownHosts = readKnownHosts(localKnownHosts.copy(), remoteKnownHostsFile)
                    writeKnownHosts(remoteKnownHosts, remoteKnownHostsFile)
                else:
                    remoteKnownHostsFile = GV.known_hosts_fname
                
                remoteIdentityPubFile = os.path.join(tempDir, h.host(), 'id_rsa.pub')
                if os.path.exists(remoteIdentityPubFile):
                    if not filecmp.cmp(GV.id_rsa_pub_fname, remoteIdentityPubFile):
                        print '  ... retaining identity from %s' % h.host()
                    remoteIdentity = ""
                    remoteIdentityPub = ""
                else:
                    remoteIdentity = GV.id_rsa_fname
                    remoteIdentityPub = GV.id_rsa_pub_fname
                
                cmd = ('scp -q -o "BatchMode yes" -o "NumberOfPasswordPrompts 0" ' +
                       '%s %s %s %s %s:.ssh/ 2>&1' 
                       % (remoteAuthKeysFile, 
                          remoteKnownHostsFile, 
                          remoteIdentity, 
                          remoteIdentityPub, 
                          canonicalize( h.host() )))
                h.popen(cmd)
                
    except:
        errmsg = '[ERROR] cannot complete key exchange: %s' % sys.exc_info()[0]
        print >> sys.stderr, errmsg
        raise
    
    finally:
        for h in GV.allHosts:
            if not h.isPclosed():
                (ok, content) = h.pclose()
                if ok: 
                    print '  ... finished key exchange with', h.host()
                else:
                    errmsg = "[ERROR] unable to copy authentication files to %s" % h.host()
                    print >> sys.stderr, errmsg
                    for line in content.splitlines():
                        print >> sys.stderr, '    ', line
    
    if errmsg: sys.exit(1)

    print; print '[INFO] completed successfully'
    sys.exit(0)

except KeyboardInterrupt:
    sys.exit('\n\nInterrupted...')

finally:
    # Discard the temporary working directory (borrowed from Python 
    # doc for os.walk).
    if tempDir and not os.environ.has_key('KEEPTEMP'):
        if GV.opt['-v']: print '[INFO] deleting tempDir %s' % tempDir
        for root, dirs, files in os.walk(tempDir, topdown=False):
            for name in files:
                os.remove(os.path.join(root, name))
            for name in dirs:
                os.rmdir(os.path.join(root, name))
        os.rmdir(tempDir)
        
    coverage.stop()
    coverage.generate_report()

