#!/usr/bin/env python

# Copyright (C) 2006-2007 Lattica, Inc.
#
# SafeKeep is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 2 of the License, or
# (at your option) any later version.
#
# Safekeep is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Safekeep.  If not, see <http://www.gnu.org/licenses/>.

from __future__ import generators
import getopt, os, os.path, popen2, re, sys, fnmatch
import commands, tempfile, time, traceback
import getpass, pwd, xml.dom.minidom
import socket, smtplib

######################################################################
# Python 2.2 compatibility
######################################################################
# There is no guarantee that we'll continue supporting Python 2.2
# indefinitely, but we make a reasonable effor to do so as long as
# it doesn't result in major complication/ugliness.

try:
    True, False
except NameError:
    True, False = 1, 0

def enumerate(obj):
    i = -1
    for item in obj:
        i += 1
        yield i, item

######################################################################
# Global settings
######################################################################

config_file = '/etc/safekeep/safekeep.conf'
config_ext = '.backup'
logbuf = []
is_client = False
verbosity_level = 1
work_user = getpass.getuser()
backup_user = None
home_dir = None
base_dir = None

PROTOCOL = "1.1"
VERSION = "1.0.5"
VEBOSITY_BY_CLASS = {'DBG': 3, 'INFO': 2, 'WARN': 1, 'ERR': 0}

######################################################################
# Miscellaneous support functions
######################################################################

def send(msg):
    print msg
    sys.stdout.flush()

def log(msg, cls=None):
    global logbuf
    if cls:
        if is_client: cls = cls.lower()
        msg = '%s: %s' % (cls, msg)
    else:
        for c in VEBOSITY_BY_CLASS.keys():
            if msg.startswith(c + ': '):
                cls = c
                break
        else:
            cls = 'UNK'

    cutoff = VEBOSITY_BY_CLASS.get(cls.upper())
    if cutoff is None: cutoff = 3
    if is_client or verbosity_level >= cutoff:
        logbuf.append(msg)
        if is_client:
            print msg
            sys.stdout.flush()
        else:
            print >> sys.stderr, msg

def info_file(file, marker=None):
    info('## File: ' + file)
    errs = 0;
    fin = open(file, 'r')
    try:
        for line in fin.readlines():
            if marker:
                if line.startswith(marker):
                    marker = None
                continue
            if (line.startswith("Errors ")):
                errs = int(line[6:])
            info(line.rstrip())
    finally:
        fin.close()
    return errs

def debug(msg):
    log(msg, 'DBG')

def info(msg):
    log(msg, 'INFO')

def warn(msg):
    log(msg, 'WARN')

def error(msg):
    log(msg, 'ERR')

def spawn(args):
    if isinstance(args, str) or isinstance(args, unicode):
        debug('Run [' + args + ']')
        cmd = args.split(' ')[0]
    else:
        debug('Run [' + ' '.join(args) + ']')
        cmd = args[0]
    proc = popen2.Popen4(args)
    proc.tochild.close()
    for line in proc.fromchild:
        info(line.rstrip())
    proc.fromchild.close()
    rc = proc.wait()
    if os.WIFEXITED(rc):
        if os.WEXITSTATUS(rc) == 0:
            ret = None
        else:
            ret = 'exited with non zero status: %d' % os.WEXITSTATUS(rc)
    elif os.WIFSIGNALED(rc):
        ret = 'killed by signal: %d' % os.WTERMSIG(rc)
    elif os.WCOREDUMP(rc):
        ret = 'coredumped'
    else:
        ret = 'unknown exit status: %d' + rc
    if ret:
        error('%s failed: %s' % (cmd, ret));
    return ret

def send_notification(email, smtp):
    info('Sending email to %s via %s' % (','.join(email), smtp))
    global logbuf
    hostname = socket.gethostname()
    msg = 'From: SafeKeep@' + hostname + \
          '\r\nTo: ' + ', '.join(email) + \
          '\r\nSubject: SafeKeep results for ' + hostname + \
          '\r\n\r\n' + '\r\n'.join(logbuf)
    if smtp:
        server = smtplib.SMTP(smtp)
        server.sendmail('SafeKeep@' + hostname, email, msg)
        server.quit()
    else:
        cmd = '/usr/sbin/sendmail -t'
        pin = os.popen(cmd, 'w')
        try:
            pin.write(msg)
        finally:
            pin.close()

def is_temp_root(dir):
    return dir != '/'

def reroot(root, path):
    if root == '/': return path
    if root.endswith('/'): root = root[:-1]
    if not path: return root
    if path.startswith('/'): return root + path
    return os.path.join(root, path)

def parse_prop_file(file):
    props = {}
    fin = open(file)
    lines = fin.readlines()
    fin.close()
    for line in lines:
        line = line.strip()
        if len(line) is 0 or line[0] is '#': continue
        if '=' in line: 
            key, value = line.split('=', 1)
            props[key.strip()] = value.strip()
        else:
            props[line] = None
    return props            

######################################################################
# Configuration file parser
######################################################################

class ConfigException (Exception):
    def __init__(self, value):
        self.value = value
    def __str__(self):
        return repr(self.value)

def parse_dump(dump_el):
    type = dump_el.getAttribute('type')
    if not type:
        raise ConfigException('You need to specify the database type')
    if type not in ('postgres', 'postgresql', 'pgsql', 'mysql'):
        raise ConfigException('Invalid database type: ' + type)
    db = dump_el.getAttribute('db')
    user = dump_el.getAttribute('user')
    dbuser = dump_el.getAttribute('dbuser')
    dbpasswd = dump_el.getAttribute('dbpasswd')
    file = dump_el.getAttribute('file')
    if not file:
        raise ConfigException('You need to specify where the database should be dumped')
    cleanup = dump_el.getAttribute('cleanup')
    return { 'type' : type, 'db' : db, 'user' : user, 'dbuser' : dbuser, 'dbpasswd': dbpasswd,
             'file' : file, 'cleanup' : cleanup }

def parse_snap(snap_el):
    device = snap_el.getAttribute('device')
    if not device:
        raise ConfigException('Please specify the device to be snapshot')
    if device.rfind('/') == -1 or device.endswith('/'):
        raise ConfigException('The device name seems incorrect: ' + device)
    size = snap_el.getAttribute('size')
    if not size:
        raise ConfigException('Please specify the size for the snapshot')
    return { 'device' : device, 'size' : size }

def parse_clude(clude_el):
    path = clude_el.getAttribute('path')
    path = path.replace('*', '\*').replace('?', '\?')
    path = path.replace('[', '\[').replace(']', '\]')
    glob = clude_el.getAttribute('glob')
    regexp = clude_el.getAttribute('regexp')
    if not path and not glob and not regexp:
        raise ConfigException('Empty ' + clude_el.tagName)
    return { 'type' : clude_el.tagName, 'path' : path, 'glob' : glob, 'regexp' : regexp }

def parse_config(backup_el, dflt_id):
    if backup_el.tagName != 'backup':
        raise ConfigException('Invalid config file, the top level element must be <backup>')
    id = backup_el.getAttribute('id')
    if not id: id = dflt_id

    host_el = backup_el.getElementsByTagName('host')
    if host_el:
        host = host_el[0].getAttribute('name')
        user = host_el[0].getAttribute('user')
        key_ctrl = host_el[0].getAttribute('key-ctrl')
        key_data = host_el[0].getAttribute('key-data')
    else:
        host = user = key_ctrl = key_data = None
    if host and not user:
        user = 'root'
    if host and not key_ctrl:
        key_ctrl = os.path.join('.ssh', 'safekeep-server-ctrl-key')
    if host and not key_data:
        key_data = os.path.join('.ssh', 'safekeep-server-data-key')
    if key_ctrl and not os.path.isabs(key_ctrl):
        key_ctrl = os.path.join(home_dir, key_ctrl)
    if key_data and not os.path.isabs(key_data):
        key_data = os.path.join(home_dir, key_data)

    repo_el = backup_el.getElementsByTagName('repo')
    dir = None
    retention = None
    if len(repo_el) == 1:
        dir = repo_el[0].getAttribute('path')
        retention = repo_el[0].getAttribute('retention')
    elif len(repo_el) > 1:
        raise ConfigException('Can not have more than a repo element')
    if not dir: dir = id
    dir = os.path.join(base_dir, dir)

    options_els = backup_el.getElementsByTagName('options')
    options = []
    if len(options_els) > 0:
        for options_el in options_els[0].childNodes:
            if options_el.nodeType != options_el.ELEMENT_NODE:
                continue
            option = options_el.nodeName
            if option in ('special-files', 'rdiff-backup'):
                if options_el.hasAttributes():
                    for key, value in options_el.attributes.items():
                        options.append({ option : { key : value } })
                else:
                    raise ConfigException('Option "%s" has no value' % option)
            else:
                raise ConfigException('Unknown option "%s"' % option)

    setup_el = backup_el.getElementsByTagName('setup')
    dumps = []
    snaps = []
    if len(setup_el) > 0:
        dump_els = setup_el[0].getElementsByTagName('dump')
        for dump_el in dump_els:
            dumps.append(parse_dump(dump_el))
        snap_els = setup_el[0].getElementsByTagName('snapshot')
        for snap_el in snap_els:
            snaps.append(parse_snap(snap_el))

    data_el = backup_el.getElementsByTagName('data')
    
    if data_el:
        child_els = data_el[0].childNodes
        cludes = []
        for child_el in child_els:
            if child_el.nodeType != child_el.ELEMENT_NODE:
                continue
            if child_el.tagName not in ('include', 'exclude'):
                continue
            cludes.append(parse_clude(child_el))
        cludes.append({ 'type' : 'exclude', 'path' : '', 'glob' : '', 'regexp' : '.*' })
    else:
        path_xcludes = [ '/dev/', '/media/', '/mnt/', '/net/', '/proc/', '/selinux/', '/sys/',
                         '/tmp/', '/var/cache', '/var/lock', '/var/run', '/var/tmp',
                         '/var/named/chroot/dev', '/var/named/chroot/proc', 
                         '/var/named/chroot/var/run', '/var/named/chroot/var/tmp' ]
        cludes = [{ 'type' : 'exclude', 'path' : path, 'glob' : None, 'regexp' : None } for path in path_xcludes]

    return { 'id': id, 'host' : host, 'user' : user, 'key_ctrl' : key_ctrl, 'key_data' : key_data,
             'dir' : dir, 'retention' : retention, 'dumps' : dumps, 'snaps' : snaps,
             'cludes' : cludes, 'options' : options}

def parse_locs(cfglocs):
    cfgfiles = []
    for cfg in cfglocs:
        if os.path.isdir(cfg):
            for ent in os.listdir(cfg):
                if not ent.endswith(config_ext):
                    warn('Ignoring file %s not ending in %s' % (os.path.join(cfg, ent), config_ext))
                    continue
                filepath = os.path.join(cfg, ent)
                if not os.path.isfile(filepath):
                    continue
                cfgfiles.append(filepath)
        elif os.path.isfile(cfg):
            cfgfiles.append(cfg)
        else:
            warn('Inaccessible configuration, ignoring: %s' % cfg)

    cfgs = {}
    for filepath in cfgfiles:
        filename = os.path.splitext(os.path.basename(filepath))[0]

        cfg_file = open(filepath)
        cfg_str = cfg_file.read().strip()
        cfg_file.close()

        dom = xml.dom.minidom.parseString(cfg_str)
        try:
            cfg = parse_config(dom.documentElement, filename)
        finally:
            dom.unlink()
        cfg['text'] = cfg_str
        if cfg['id'] in cfgs:
            raise ConfigException('Duplicate client ID: %s' % cfg['id'])
        cfgs[cfg['id']] = cfg

    return cfgs

######################################################################
# DB and SNAPSHOT support
#   setup methods can raise exception to signal errors
#   teardown methods must succeed and cleanup the state
######################################################################

def do_client_dbdump(cfg):
    debug('Doing DB dumps')
    for dump in cfg['dumps']:
        type = dump['type']
        if type in ('postgres', 'postgresql', 'pgsql'):
            if dump['db']:
                args = ['pg_dump']
            else:
                args = ['pg_dumpall']
            if dump['dbuser']:
                args.extend(['-U', dump['dbuser']])
            if dump['db']:
                args.append(dump['db'])
        elif type in ('mysql'):
            args = ['mysqldump']
            if dump['dbuser']:
                args.extend(['-u', dump['dbuser']])
            if dump['dbpasswd']:
                args.extend(['-p', dump['dbpasswd']])
            if dump['db']:
                args.append(dump['db'])
            else:
                args.append('-A')
        else:
            warn('Invalid database type: ' + type)
            continue
        if dump['user']:
            cmd = ' '.join([commands.mkarg(arg) for arg in args])
            args = [ 'su', '-c', cmd, '-', dump['user'] ]
        cmd = ' '.join([commands.mkarg(arg) for arg in args])
        cmd = '%s > %s' % (cmd, commands.mkarg(dump['file']))
        ec = spawn(cmd)
        if ec:
            warn('Can not dump the database: ' + dump['db'])

def do_client_dbdump_teardown(cfg):
    debug('Tear down DB dumps')
    for dump in cfg['dumps']:
        if dump['cleanup'] != 'true':
            continue
        try:
            os.remove(dump['file'])
        except Exception, e:
            warn('Unable to remove dump file: %s for database %s because: %s' %
                 (dump['file'], dump['db'], e))

def lvm_snap_information():
    (cin, cout) = os.popen4(['lvs', '--separator', ':', '--noheadings'])
    lines = cout.readlines()
    cout.close()
    cin.close()
    lvms = []
    for line in lines:
        if line.count(':') > 3:
            (volume, group, attr, blah1) = line.lstrip().split(':', 3)
            if fnmatch.fnmatch(volume, '*_snap_safekeep-*') and attr[0].lower() == 's':
                lvms.append([volume, group])
    return lvms

def mount_information(reverse = False):
    (cin, cout) = os.popen4('mount')
    lines = cout.readlines()
    cout.close()
    cin.close()
    mounts = []
    if reverse:
        lines.reverse()
    for line in lines:
        (device, blah1, mountpoint, blah2, mounttype, mountoptions) = line.split()
        mounts.append([device, mountpoint, mounttype, mountoptions[1:-1]])
    return mounts

def map_lvm_device(device):
    device = device.replace('/mapper','').replace('-','/')
    return device.split('/')[-2:]

def check_lvm_information(device):
    (group, volume) = map_lvm_device(device)
    for (lvm_volume, lvm_group) in lvm_snap_information():
        if lvm_group == group and lvm_volume.startswith(volume):
            return True
    return False

def gather_lvm_information(device):
    (group, volume) = map_lvm_device(device)
    for (device, mountpoint, mounttype, mountoptions) in mount_information(False):
        if [group, volume] == map_lvm_device(device):
            return (group, volume, mountpoint, mounttype)
    return (None, None, None, None)

def gather_snap_information(device, bdir):
    (group, volume, mountpoint, mounttype) = gather_lvm_information(device)
    if not mountpoint: return (None, None, None, None)
    lvmdev =  os.path.join('/dev', group, volume)
    if bdir[-1] == '/': bdir = bdir[:-1]
    snapname = '%s_snap_%s' % (volume, os.path.basename(bdir))
    snapdev = os.path.join('/dev', group, snapname)
    if os.path.isabs(mountpoint[0]): mountpoint = mountpoint[1:]
    return (lvmdev, snapdev, os.path.join(bdir, mountpoint), mounttype)

def do_client_snap(cfg, bdir):
    assert is_temp_root(bdir)
    debug('Doing FS snapshots')
    for snap in cfg['snaps']:
        device = snap['device']
        (lvmdev, snapdev, snapmnt, snaptyp) = gather_snap_information(device, bdir)
        if not snapmnt:
            warn('Cannot find the mountpoint for: ' + device)
            continue
        args = ['lvcreate', '--snapshot', '--size', snap['size'],
                '--name', os.path.basename(snapdev), lvmdev]
        ec = spawn(args)
        if ec:
            warn('Can not snapshot the device: ' + device)
            continue
        # no need to mkdir since the mountpoint already exists
        args = ['mount', '-t', snaptyp, snapdev, snapmnt]
        ec = spawn(args)
        if ec:
            warn('Can not mount the snapshot: ' + device)
            ret = spawn(['lvremove', '--force', snapdev])
            if ret:
                warn('Can not tear down snapshot: ' + device)

def do_client_snap_teardown(cfg, bdir):
    assert is_temp_root(bdir)
    debug('Tear down FS snapshots dumps')
    snaps = list(cfg['snaps'])
    snaps.reverse()
    for snap in snaps:
        device = snap['device']
        (lvmdev, snapdev, snapmnt, snaptyp) = gather_snap_information(device, bdir)
        if not snapmnt:
            warn('Can not find the mountpoint for: ' + device)
            continue
        ret = spawn(['umount', snapmnt])
        if ret:
            warn('Can not umount the snapshot: %s' % snapmnt)
        ret = spawn(['lvremove', '--force', snapdev])
        if ret:
            warn('Can not tear down snapshot: ' + device)

######################################################################
# Client implementation
######################################################################

def do_client_config(cmd):
    cfgStr = ''

    (cfg_cmd, cnt_str, dflt_id) = cmd.split(':', 2)
    for i in xrange(int(cnt_str)):
        line = sys.stdin.readline()
        if not line: raise ConfigException('Unexpected end of file')
        cfgStr += line

    dom = xml.dom.minidom.parseString(cfgStr)
    try:
        return parse_config(dom.documentElement, dflt_id)
    finally:
        dom.unlink()

def do_client_setup(cfg):
    debug('Do setup of %s' % cfg['host'])

    do_client_dbdump(cfg)

    if len(cfg['snaps']) > 0:
        debug('Checking FS snapshots')
        for snap in cfg['snaps']:
            device = snap['device']
            if check_lvm_information(device):
                raise Exception("Previous snapshots found for %s: run 'safekeep --server --cleanup' to correct" % device)

        ret = spawn(['modprobe', 'dm-snapshot'])
        if ret:
            warn('modprobe dm-snapshot failed, continuing')
        bdir = tempfile.mkdtemp("-rbind", "safekeep-", "/mnt")
        ret = spawn(['mount', '--rbind', '/', bdir])
        if ret:
            warn('mount --rbind failed, snapshotting will be disabled')
            try:
                os.rmdir(bdir)
            except Exception, e:
                warn('Failed to remove: ' + bdir)
            bdir = '/'
        else:
            do_client_snap(cfg, bdir)
    else:
        bdir = '/'
    debug('Working root is %s' % bdir)

    return bdir

def do_client_cleanup(cfg, bdir):
    debug('Do cleanup of %s in %s' % (cfg['host'], bdir))
    if is_temp_root(bdir):
        do_client_snap_teardown(cfg, bdir)

        ret = spawn(['umount', '-l', bdir])
        if ret:
            warn('Failed to unmount: ' + bdir)
        else:
            try:
                os.rmdir(bdir)
            except Exception, e:
                warn('Unable to remove: ' + bdir)

    do_client_dbdump_teardown(cfg)

def do_client_compat(server_versions):
    debug('Server versions: %s' % server_versions)

def do_client_scrub():
    debug("Do client scrub loop")

    if os.getuid():
        if is_client:
            raise Exception('client not running as root')
        else:
            warn('--cleanup should be run as root on client')
            info('No cleanup performed')
    else:
        scrubbed = False

        if os.environ['PATH'][-1] == ':':
            os.environ['PATH'] += '/sbin:/usr/sbin:/usr/local/sbin:'
        else:
            os.environ['PATH'] += ':/sbin:/usr/sbin:/usr/local/sbin'

        # Go through and unmount anythings that are still hanging around

        debug("Cleaning up existing mounts")
        for (device, mountpoint, mounttype, mountoptions) in mount_information(True):
            if mountpoint.startswith('/mnt/safekeep-'):
                info("Removing mount %s" % mountpoint)
                if device == '/' and 'bind' in mountoptions.split(','):
                    info("Removing rbind directory %s" % mountpoint)
                    ret = spawn(['umount', '-l', mountpoint])
                    if ret:
                        warn('Failed to unmount: ' + mountpoint)
                    else:
                        try:
                            os.rmdir(mountpoint)
                        except Exception, e:
                            warn('Failed to remove: ' + mountpoint)
                else:
                    ret = spawn(['umount', mountpoint])
                    if ret:
                        warn('Can not unmount the snapshot: %s' % mountpoint)
                if fnmatch.fnmatch(device, '*_snap_safekeep-*'):
                    info("Removing snapshot %s" % device)
                    ret = spawn(['lvremove', '--force', device])
                    if ret:
                        warn('Can not tear down snapshot: ' + device)
                scrubbed = True

        # Now cleanup any snapshots still hanging around

        debug("Cleaning up remaining snapshots")
        for (volume, group) in lvm_snap_information():
            device = os.path.join('/dev', group, volume)
            info("Removing snapshot %s" % device)
            ret = spawn(['lvremove', '--force', device])
            if ret:
                warn('Can not tear down snapshot: ' + device)
            scrubbed = True

        # Now cleanup any safekeep directories still hanging around

        debug("Cleaning up remaining safekeep directories")
        if os.path.isdir('/mnt'):
            for ent in os.listdir('/mnt'):
                mountpoint = os.path.join('/mnt', ent)
                if ent.startswith('safekeep-') and os.path.isdir(mountpoint):
                    info("Removing rbind directory %s" % mountpoint)
                    try:
                        os.rmdir(mountpoint)
                    except Exception, e:
                        warn('Failed to remove: ' + mountpoint)

        if not scrubbed:
            info('No cleanup required')

def do_client():
    debug("Do client main loop")
    should_cleanup = True
    bdir = '/'
    try:
        while True:
            try:
                line = sys.stdin.readline()
                if line.startswith('ALOHA'):
                    do_client_compat(line.split(':', 1)[1])
                    send('OK %s, %s' % (PROTOCOL, VERSION))
                elif line.startswith('CONFIG'):
                    cfg = do_client_config(line)
                    send('OK')
                elif line.startswith('SETUP'):
                    bdir = do_client_setup(cfg)
                    send('OK ' + bdir)
                elif line.startswith('CLEANUP'):
                    dir = line[7:].strip()
                    if dir == bdir: should_cleanup = False
                    do_client_cleanup(cfg, dir)
                    send('OK')
                elif line.startswith('SCRUB'):
                    do_client_scrub()
                    send('OK')
                elif not line:
                    break
                else:
                    send('ERROR Unknown command: ' + line)
                    break
            except Exception, e:
                traceback.print_exc(file=sys.stdout)
                send('ERROR %s' % e)
    finally:
        if should_cleanup:
            do_client_cleanup(cfg, bdir)


######################################################################
# Server implementation
######################################################################

def do_server_getanswer(cout):
    while True:
        line = cout.readline()
        if line.startswith('OK'):
            return line[2:-1].strip()
        elif line.startswith('ERROR'):
            raise Exception(line[5:].strip())
        elif not line:
            raise Exception('client died unexpectedly')
        else:
            log(line[:-1])

def do_server_rdiff(cfg, bdir, force):
    args = ['rdiff-backup']

    if cfg['host']:
        schema = 'ssh -C -i %s %%s rdiff-backup --server' % (cfg['key_data'])
        args.extend(['--remote-schema', schema])

    if force:
        args.extend(['--force'])

    options_append = []
    special_files = ['--exclude-device-files', '--exclude-sockets', '--exclude-fifos']
    for option in cfg['options']:
        if 'special-files' in option:
            if 'include' in option['special-files']:
                if 'true' == option['special-files']['include'].lower():
                    special_files = ['--include-special-files']

        # Note if we ever add other backends this section should only be run
        # when rback-diff is the current option.

        if 'rdiff-backup' in option:
            if 'append' in option['rdiff-backup']:
                options_append.extend(option['rdiff-backup']['append'].split(None))

    args.extend(special_files)
    args.extend(options_append)

    for clude in cfg['cludes']:
        opt = '--' + clude['type']
        if clude['path']:
            args.extend([opt, reroot(bdir, clude['path'])])
        if clude['glob']:
            args.extend([opt, reroot(bdir, clude['glob'])])
        if clude['regexp']:
            args.extend([opt + '-regexp', bdir + clude['regexp']])

    userhost = ''
    if cfg['host']:
        userhost = '%s@%s' % (cfg['user'], cfg['host'])
    args.extend([userhost + '::' + bdir, cfg['dir']])
    ret = spawn(args)
    if ret:
        raise Exception('Failed to run rdiff-backup')

def do_server_rdiff_cleanup(cfg):
    args = ['rdiff-backup', '--check-destination-dir', cfg['dir']]
    ret = spawn(args)
    if ret:
        warn('Failed to cleanup old data, please fix the problem manually')

def do_server_data_cleanup(cfg):
    args = ['rdiff-backup', '--force', '--remove-older-than', cfg['retention'], cfg['dir']]
    ret = spawn(args)
    if ret:
        warn('Failed to cleanup old data, please fix the problem manually')

def do_server_compat(client_versions):
    (client_protocol, client_version) = client_versions.split(',')
    (client_major, client_minor) = client_protocol.strip().split('.')
    (server_major, server_minor) = PROTOCOL.split('.')
    if server_major != client_major:
        raise Exception('Incompatible protocols: %s <> %s' % (PROTOCOL, client_protocol))
    elif server_minor > client_minor:
        warn('Protocol mismatch: %s <> %s' % (PROTOCOL, client_protocol))

def do_server(cfgs, ids, force, cleanup):
    debug("Do server main loop")
    for cfg in cfgs.itervalues():
        id = cfg['id']
        if ids and id not in ids: continue
        info('------------------------------------------------------------------')
        info('Server backup starting for client %s' % id)

        cleaned_up = 0
        try:
            if cfg['host']:
                if not os.path.isfile(cfg['key_ctrl']):
                    raise Exception('Client %(id)s missing ctrl key %(key_ctrl)s' % cfg)
                if not os.path.isfile(cfg['key_data']):
                    raise Exception('Client %(id)s missing data key %(id)s' % cfg)

            datadir = os.path.join(os.getcwd(), cfg['dir'])
            if not os.path.isdir(datadir):
                try:
                    os.makedirs(datadir)
                except EnvironmentError, ex:
                    raise Exception('Can not create data store dir: %s' % datadir)

            rdiff_logdir = os.path.join(datadir, 'rdiff-backup-data')
            if cfg['retention'] and os.path.isdir(rdiff_logdir) and not cleanup:
                do_server_data_cleanup(cfg)

            if cfg['host']:
                cmd = 'ssh -T -i %(key_ctrl)s -l %(user)s %(host)s safekeep --client' % cfg
            else:
                cmd = 'safekeep --client'
            debug('Run [' + cmd + ']')
            (cin, cout) = os.popen4(cmd)

            cin.write('ALOHA: %s, %s\n' % (PROTOCOL, VERSION))
            cin.flush()
            client_versions = do_server_getanswer(cout)
            do_server_compat(client_versions)

            cin.write('CONFIG: %d: %s\n' % (len(cfg['text'].splitlines()), id))
            cin.write(cfg['text'] + '\n')
            cin.flush()
            do_server_getanswer(cout)
            if cleanup:
                cin.write('SCRUB\n')
                cin.flush()
                do_server_getanswer(cout)
                bdir = '/'  # Fake directory for the rest of the cleanup
                do_server_rdiff_cleanup(cfg)
                cleaned_up = 1
                errs = 0
            else:
                cin.write('SETUP\n')
                cin.flush()
                bdir = do_server_getanswer(cout)

                if os.path.isdir(rdiff_logdir):
                    rdiff_logpre = os.listdir(rdiff_logdir)
                else:
                    rdiff_logpre = []

                backup_log = os.path.join(rdiff_logdir, 'backup.log')
                if os.path.isfile(backup_log):
                    backup_marker = '=== Backup session on %s ===' % time.asctime()
                    fbm = open(backup_log, 'a')
                    fbm.write(backup_marker + '\n')
                    fbm.close()
                else:
                    backup_marker = None

                do_server_rdiff(cfg, bdir, force)

                errs = 0
                if os.path.isdir(rdiff_logdir):
                    info_file(backup_log, backup_marker)
                    rdiff_logpost = os.listdir(rdiff_logdir)
                    for lfn in rdiff_logpost:
                        if lfn.startswith('session_statistics.') and lfn.endswith('.data') and lfn not in rdiff_logpre:
                            errs += info_file(os.path.join(rdiff_logdir, lfn))
                else:
                    warn('Log dir does not exist.')

            cin.write('CLEANUP %s\n' % bdir)
            cin.flush()
            do_server_getanswer(cout)

            if errs == 0:
                info('Server backup for client %s: OK' % id)
            else:
                info('Server backup for client %s: OK (%d WARNINGS)' % (id, errs))

        except Exception, e:
            if cleanup and not cleaned_up:
                info('Client-side cleanup for client %s: FAILED' % id)
                do_server_rdiff_cleanup(cfg)
            else:
                error(e)
                error('Server backup for client %s: FAILED' % id)

    info('------------------------------------------------------------------')
    debug('Server backup done')

def do_list(cfgs, ids, list_type, list_date, list_parsable):
    debug("Do server listing main loop")
    for cfg in cfgs.itervalues():
        id = cfg['id']
        if ids and id not in ids: continue
        if list_parsable:
            info('Client: %s' % id)
        else:
            info('------------------------------------------------------------------')
            info('Server listing for client %s' % id)
            

        args = ['rdiff-backup']

        if list_type is 'increments':
            args.extend(['--list-increments'])
        elif list_type is 'sizes':
            args.extend(['--list-increment-sizes'])
        elif list_type is 'changed':
            args.extend(['--list-changed-since', list_date])
        elif list_type is 'attime':
            args.extend(['--list-at-time', list_date])
        else:
            assert False, 'Unknown list type: ' + list_type

        if list_parsable:
            args.extend(['--parsable-output'])

        args.extend([cfg['dir']])
        ret = spawn(args)
        if ret:
            raise Exception('Failed to run rdiff-backup')

    if not list_parsable:
        info('------------------------------------------------------------------')
    debug('Server listing done')

def do_keys(cfgs, ids, identity, status, dump, deploy):
    for cfg in cfgs.itervalues():
        id = cfg['id']
        if ids and id not in ids: continue
        info('Handling keys for client: %s' % id)
        if not cfg['host']:
            info('%s: Client is local, it needs no keys' % id)
            continue
        cmds = ['safekeep --client', 'rdiff-backup --server --restrict-read-only /']
        privatekeyfiles = [cfg.get('key_ctrl'), cfg.get('key_data')]
        lines = []
        keys_ok = False
        for (cmd, privatekeyfile) in zip(cmds, privatekeyfiles):
            publickeyfile = privatekeyfile + '.pub'
            if not os.path.isfile(privatekeyfile):
                if os.path.isfile(publickeyfile):
                    error('%s: Public key exists %s, but private key is missing. Skipping client.' % (id, publickeyfile))
                    break
                if dump:
                    print '%s: Key does not exist: %s.' % (id, privatekeyfile)
                    break
                if status:
                    print '%s: Key does not exist: %s. Will be generated.' % (id, privatekeyfile)
                    break
                if deploy:
                    info('%s: Key do not exist, generating it now: %s' % (id, privatekeyfile))
                    gencmd = 'ssh-keygen -q -b 1024 -t dsa -N "" -C "SafeKeep auto generated key at %s@%s" -f %s' % (backup_user, os.uname()[1], privatekeyfile)
                    if backup_user is not work_user:
                        gencmd = 'su -s /bin/sh -c %s - %s' % (commands.mkarg(gencmd), backup_user)
                    debug(gencmd)
                    if os.system(gencmd):
                        error('%s: Failed to generate key %s. Skipping client.' % (id, privatekeyfile))
                        break
            if not os.path.isfile(publickeyfile):
                error('%s: Private key exists %s, but public key is missing. Skipping client.' % (id, privatekeyfile))
                break
            fin = open(publickeyfile, 'r')
            publickey = fin.read()
            fin.close()
            line = 'command="%s",no-port-forwarding,no-X11-forwarding,no-agent-forwarding,no-pty %s' % (cmd, publickey.strip())
            lines.append(line)
        else:
            keys_ok = True

        if not keys_ok:
            continue

        output = '\n'.join(lines)
        if dump:
            print output

        basessh = 'ssh'
        if identity: basessh += ' -i %s' % (commands.mkarg(identity))

        if status or deploy:
            cmd = '%s %s@%s "if test -f .ssh/authorized_keys; then cat .ssh/authorized_keys; fi"' % (basessh, cfg['user'], cfg['host'])
            debug(cmd)
            out = os.popen(cmd, 'r')
            authtext = out.read()
            if out.close():
                warn('%s: Failed to read the authorized_keys file.' % id)
            auth_keys = parse_authorized_keys(authtext)
            this_keys = parse_authorized_keys(output)
            new_keys = []
            for this_key in this_keys:
                for auth_key in auth_keys:
                    if this_key[2] == auth_key[2]: break
                else:
                    new_keys.append(this_key)
            if not new_keys:
                if status:
                    print '%s: Client is up to date.' % id
                continue

            if status:
                print '%s: Keys will be deployed on the client.' % id
            if deploy:
                cmd = '%s %s@%s "umask 077; test -d .ssh || mkdir .ssh; cat >> .ssh/authorized_keys"' % (basessh, cfg['user'], cfg['host'])
                debug(cmd)
                pipe = os.popen(cmd, 'w')
                pipe.write('%s\n' % '\n'.join([key[4] for key in new_keys]))
                if pipe.close():
                    error('Failed to deliver the keys to the client')


# parses authozied_keys, see sshd(8) man page for details
def parse_authorized_keys(keystext):
    keys = []
    for line in keystext.splitlines():
        line = line.strip()
        if not line or line[0] == '#': continue
        if line[0] in '0123456789':
            warn('SSH Protocol 1 keys are ignored: %s' % line)
            continue
        opts =''
        if line[0:7] not in ('ssh-dss', 'ssh-rsa'):
            in_str = False
            in_esc = False
            for i, c in enumerate(line):
                if in_str:
                    if in_esc: in_esc = False
                    elif c is '\'': in_esc = True
                    elif c is '"': in_str = False
                else:
                    if c is ' ':
                        rest = line[i:].strip()
                        break
                    elif c is '"': in_str = True
                opts += c
            else:
                info('Invalid key line, ignoring: %s' % line)
                continue
        else:
            rest = line

        if rest[0] in '0123456789':
            warn('SSH Protocol 1 keys are ignored: %s' % line)
            continue

        parts = rest.split(None, 2)
        if len(parts) < 2:
            error('Invalid key line, skipping: %s' % line)
            continue

        type = parts[0]
        if type not in ('ssh-dss', 'ssh-rsa'):
            error('Invalid key type "%s", skipping: %s' % (type, line))
            continue

        base46enc = parts[1]

        if len(parts) is 2:
            comment = None
        else:
            comment = parts[2]

        keys.append((opts, type, base46enc, comment, line))

    return keys

######################################################################
# Main routine
######################################################################

def usage(exitcode=None):
    print 'usage: %s --server [common options] [server options] <client-id>*' % (sys.argv[0])
    print '       %s --keys [common options] [keys options] <client-id>*' % (sys.argv[0])
    print '       %s --list [common options] [list options] <client-id>*' % (sys.argv[0])
    print
    print 'mode selection (you must pick one):'
    print '--server            launch in server mode'
    print '--keys              launch in keys management mode'
    print '--list              list previous backup status'
    print
    print 'common options:'
    print '-c, --conf=FILE     use the FILE configuration file'
    print '-h, --help          show this help message and exit'
    print '-q, --quiet         decreases the verbosity level'
    print '-v, --verbose       increases the verbosity level'
    print '-V, --version       show the version number and exit'
    print '--noemail           disables the sending of email'
    print
    print 'server options:'
    print '--force             force backup destination overwriting, dangerous!'
    print '--cleanup           perform cleanup actions after a failure'
    print
    print 'keys options:'
    print '-i FILE             use FILE as identity for RSA/DSA authentication'
    print '--status            display the key status for the clients (default)'
    print '--print             display the authorization keys'
    print '--deploy            deploy the authorization keys'
    print
    print 'list options:'
    print '--increments        list number and dates of increments'
    print '--parsable-output   tailor output for parsing by other programs'
    print '--sizes             list sizes of all the increments'
    print '--changed=time      list files that have changed since time'
    print '--at-time=time      list files in the archive at given time'
    if exitcode is not None: sys.exit(exitcode)

def main():
    try:
        opts, args = getopt.getopt(sys.argv[1:], 'c:e:i:hs:qvV',
                                   [ 'conf=', 'client', 'clientid=', 'deploy',
                                     'email=', 'force', 'help', 'keys',
                                     'list', 'increments', 'sizes',
                                     'parsable-output', 'changed=', 'at-time=',
                                     'noemail', 'cleanup',
                                     'print', 'quiet', 'server', 'smtp=',
                                     'status', 'verbose', 'version'])
    except getopt.GetoptError:
        usage(2)

    global backup_user, home_dir, base_dir
    mode = None
    email = []
    smtp = None
    cfgfile = None
    cfglocs = []
    verbosity = 0
    clientid = None
    force = 0
    cleanup = 0
    noemail = 0
    list_type = None
    list_parsable = 0
    list_date = None
    identity = None
    keys_status = None
    keys_print = None
    keys_deploy = None
    for o, a in opts:
        if o in ('-c', '--conf'):
            if os.path.isdir(a) or a.endswith(config_ext):
                warn('Adding client config files/dirs via this switch is deprecated')
                cfglocs.append(a)
            elif cfgfile is None:
                cfgfile = a
            else:
                error('A main configuration file can be specified only once!')
                sys.exit(2)
        elif o in ('-e', '--email'):
            warn('The -e/--email options are deprecated and will be removed in the future')
            warn('Please use the /etc/safekeep/safekeep.conf file instead')
            email.append(a)
        elif o in ('-h', '--help'):
            usage(0)
        elif o in ('-s', '--smtp'):
            warn('The -s/--smtp options are deprecated and will be removed in the future')
            warn('Please use the /etc/safekeep/safekeep.conf file instead')
            smtp = a
        elif o in ('--server', ):
            if mode: usage(2)
            mode = 'server'
        elif o in ('--list', ):
            if mode: usage(2)
            mode = 'list'
        elif o in ('--client', ):
            if mode: usage(2)
            mode = 'client'
        elif o in ('--keys', ):
            if mode: usage(2)
            mode = 'keys'
        elif o in ('--force', ):
            force = 1
        elif o in ('--cleanup', ):
            cleanup = 1
        elif o in ('--noemail', ):
            noemail = 1
        elif o in ('--increments', ):
            if list_type: usage(2)
            list_type = 'increments'
        elif o in ('--sizes', ):
            if list_type: usage(2)
            list_type = 'sizes'
        elif o in ('--parsable-output', ):
            list_parsable = 1
        elif o in ('--changed', ):
            if list_type: usage(2)
            list_type = 'changed'
            list_date = a
        elif o in ('--at-time', ):
            if list_type: usage(2)
            list_type = 'attime'
            list_date = a
        elif o in ('-i', ):
            identity = a
        elif o in ('--status', ):
            keys_status = True
        elif o in ('--print', ):
            keys_print = True
        elif o in ('--deploy', ):
            keys_deploy = True
        elif o in ('-q', '--quiet'):
            verbosity -= 1
        elif o in ('-v', '--verbose'):
            verbosity += 1
        elif o in ('-V', '--version'):
            print 'safekeep', VERSION
            return

    if mode is None:
        usage(2)

    if mode is not 'keys' and (identity or keys_status or keys_print or keys_deploy):
        usage(2)

    if mode is not 'list' and (list_type or list_date or list_parsable):
        usage(2)

    if mode is not 'server' and (email or smtp):
        usage(2)

    if not mode in ['server', 'client'] and cleanup:
        usage(2)

    if mode is 'client' and cfglocs:
        usage(2)

    if mode is not 'client':
        if cfgfile is None and os.path.isfile(config_file): 
            cfgfile = config_file
        if cfgfile and os.path.isfile(cfgfile):
            props = parse_prop_file(cfgfile)
        else:
            if cfgfile:
                warn('Configuration file does not exist, skipping: %s' % cfgfile)
            else:
                 cfgfile = config_file
            props = {}
        if 'backup.user' in props:
            backup_user = props['backup.user']
        if 'base.dir' in props:
            base_dir = props['base.dir']
        if 'email.smtp.server' in props:
            smtp = props['email.smtp.server']
        if 'email.to' in props:
            email = props['email.to'].split(',')
        if len(cfglocs) == 0: 
            locs = os.path.join(os.path.dirname(cfgfile), 'backup.d')
            if os.path.isdir(locs): cfglocs.append(locs)

    if backup_user and backup_user != work_user:
        (user, pswd, uid, gid, gecos, home_dir, shell) = pwd.getpwnam(backup_user)
        if mode is not 'keys':
            os.setregid(gid, gid)
            os.setreuid(uid, uid)
            os.environ['HOME'] = home_dir
    else:
        backup_user = work_user
        home_dir = os.getenv('HOME', '/')

    if not base_dir:
        base_dir = home_dir

    if len(cfglocs) > 0:
        cfgs = parse_locs(cfglocs)
    else:
        cfgs = {}

    if mode is 'client':
        if len(args) > 0: usage(2)
    else:
        ok = True
        for arg in args:
            if arg in cfgs: continue
            error('Unknown client ID: %s' % arg)
            if os.path.isfile(arg):
                error('It appears to be a file, configuration files are passed via the -c/--conf switch.')
            ok = False
        if not ok: sys.exit(2)

    try:
        global is_client, verbosity_level
        if mode is 'server':
            is_client = False
            verbosity_level = 1 + verbosity
            do_server(cfgs, args, force, cleanup)
        elif mode is 'list':
            if list_type is None:
                list_type = 'increments'
            is_client = False
            verbosity_level = 2 + verbosity
            do_list(cfgs, args, list_type, list_date, list_parsable)
        elif mode is 'client':
            if cleanup:
                is_client = False
                verbosity_level = 1 + verbosity
                do_client_scrub()
            else:
                is_client = True
                verbosity_level = 3 + verbosity
                do_client()
        elif mode is 'keys':
            is_client = False
            verbosity_level = 1 + verbosity
            if not keys_status and not keys_print and not keys_deploy: 
                keys_status = True
            do_keys(cfgs, args, identity, keys_status, keys_print, keys_deploy)
        else:
            assert False, 'Unknown mode: ' + mode
    except Exception, ex:
        traceback.print_exc(file=sys.stdout)
        error('ERROR: %s' % ex)

    if email and not noemail:
        send_notification(email, smtp)

if __name__ == '__main__':
    main()

# vim: et ts=8 sw=4 sts=4
