# plugs/collective.py
#
# 

""" the collective is a network of gozerbots connected by use of the bots 
 builtin webserver. nodes are urls pointing to these webservers .. this plugin
 provides the client side of the collective, to join the collective the
 webserver must be enabled and open for other connections (see WEBSERVER)
"""

__copyright__ = 'this file is in the public domain'
__depend__ = ['webserver', ]
__gendocfirst__ = ['coll-disable', 'coll-enable', 'coll-boot']

from gozerbot.generic import rlog, geturl, toenc, fromenc, waitforqueue, \
strippedtxt, handle_exception, now, geturl2
from gozerbot.rsslist import rsslist
from gozerbot.datadir import datadir
from gozerbot.pdod import Pdod
from gozerbot.persiststate import PersistState
from gozerbot.commands import cmnds
from gozerbot.plugins import plugins
from gozerbot.thr import start_new_thread
from gozerbot.plughelp import plughelp
from gozerbot.examples import examples
from gozerbot.dol import Dol
from gozerbot.config import config
from gozerbot.periodical import periodical
from gozerplugs.plugs.webserver import cfg as webcfg
from xml.sax.saxutils import unescape
from gozerbot.contrib.simplejson import loads
import Queue, time, socket, re, os

gotuuid = True
try:
    import uuid
except ImportError:
    gotuuid = False

plughelp.add('collective', 'query other gozerbots')

coll = PersistState(datadir + os.sep + 'collective')
coll.define('enable', 0)
coll.define('nodes', [])
coll.define('names', {})
coll.define('active', [])
coll.define('wait', 5)

waitre = re.compile(' wait (\d+)', re.I)

def getid(url):
    if gotuuid:
        return str(uuid.uuid3(uuid.NAMESPACE_URL, toenc(url)))
    else:
        return url

class CollNode(object):

    def __init__(self, name, url):
        self.name = name
        self.id = getid(url)
        self.url = url
        self.seen = time.time()
        self.regtime = time.time()
        self.filter = []
        self.catch = []

    def __str__(self):
        return "name=%s url=<%s> seen=%s" % (self.name, self.url, now())
        
    def cmnd(self, what, queue=None):
        if what.startswith('coll'):
            return
        try:
            what = re.sub('\s+', '+', what)
            data = geturl('%s/json?%s' % (self.url, what))
            result = loads(fromenc(data))
        except ValueError:
            # ValueError: No JSON object could be decoded
            result = ['decode error', ]
        except IOError, ex:
            result = [str(ex), ]
        except Exception, ex:
            handle_exception()
            rlog(10, 'collective', "can't fetch %s data: %s" % (self.url, \
str(ex)))
            return ['fetch error', ]
        if queue:
            queue.put((self.name, result))
        else:
            return result

    def ping(self):
        try:
            pongtxt = fromenc(geturl("%s/ping" % self.url))
        except IOError, ex:
            pongtxt = str(ex)
        except Exception, ex:
            pongtxt = ""
        return pongtxt

class Collective(object):

    def __init__(self):
        self.nodes = {}
        self.state = Pdod(datadir + os.sep + 'coll.state')
        self.startup = Pdod(datadir + os.sep + 'coll.startup')
        if not self.state.has_key('ignore'):
            self.state['ignore'] = []
        if not self.state.has_key('urls'):
            self.state['urls'] = {}
        if not self.state.has_key('names'):
            self.state['names'] = {}
        if not self.startup.has_key('start'):
            self.startup['start'] = {}

    def add(self, name, url):
        id = getid(url)
        if self.nodes.has_key(id):
            return self.nodes[id]
        self.nodes[id] = CollNode(name, url)
        self.state.set('urls', url, id)
        self.state.set('names', name,  id)
        self.persist(name, url)
        rlog(-10, 'collective', 'added %s node (%s) <%s>' % (name, id, url))
        return self.nodes[id]

    def start(self):
        for name, url in self.startup['start'].iteritems():
            self.add(name, url)
        start_new_thread(self.joinall, ())

    def persist(self, name, url):
        self.startup.set('start', name, url)
        self.startup.save()

    def get(self, id):
        try:
            return self.nodes[id]
        except KeyError:
            return

    def byname(self, name):
        id = self.state.get('names', name)
        if id:
            return self.get(id)

    def remove(self, id):
        target = self.get(id)
        if not target:
            return
        del self.state['urls'][target.url]
        del self.state['names'][target.name]
        del self.nodes[id]

    def unpersist(self, id):
        node = self.get(id)
        if not node:
            return
        try:
            del self.startup['start'][node.name]
            self.startup.save()
        except KeyError:
            pass
            
    def ignore(self, id):
        self.state['ignore'].append(id)

    def unignore(self, id):
        self.state.data['ignore'].remove(id)
        self.state.save()

    def stop(self):
        pass

    def cmndall(self, what, wait=5):
        total = {}
        queue = Queue.Queue()
        for id, node in self.nodes.iteritems():
            start_new_thread(node.cmnd, (what, queue))
        result = waitforqueue(queue, wait, maxitems=size())
        for res in result:
            total[res[0]] = res[1]
        return total

    def getnodes(self):
        return self.nodes.values()

    def getname(self, url):
        id = getid(url)
        return self.get(id).name

    def join(self, url=None):
        try:
            if url:
                result = geturl("%s/join?%s" % (url, \
webcfg.get('webport')))
            else:
                result = geturl('http://gozerbot.org:8088/join?%s' % \
webcfg.get('webport'))
        except Exception, ex:
            result = str(ex)
        result = result.strip()
        if 'added' in result:
            rlog(1, 'collective', "%s joined =>  %s" % (url, result))
        else:
            rlog(1, 'collective', "can't join %s => %s" % (url, str(result)))
        return result

    def joinall(self):
        for node in self.nodes.values():
           start_new_thread(self.join, (node.url, ))

    def boot(self, node=None):
        if node:
            self.sync(node)
        else:
            self.sync('http://gozerbot.org:8088')
        rlog(10, 'collective', 'booted %s nodes' % self.size())

    def fullboot(self):
        teller = 0
        threads = []
        for node in self.nodes.values():
            t = start_new_thread(self.sync, (node.url, False))        
            threads.append(t)
            teller += 1
        for i in threads:
            i.join()
        return teller

    def sync(self, url, throw=True):
        """ sync cacne with node """
        try:
            result = fromenc(geturl('%s/nodes' % url))
        except Exception, ex:
            if throw:
                raise
            else:
                return 0
        if not result:
            return 0
        rss = rsslist(result)
        got = 0
        for i in rss:
            try:
                url = i['url']
                id = i['id']
                name = i['name']
            except KeyError:
                continue
            try:
                self.add(name, url)
                got += 1
            except:
                handle_exception()
        return got

    def size(self):
        return len(self.nodes)

    def list(self):
        res = []
        for node in self.nodes.values():
            res.append(str(node))
        return res

    def names(self): 
        return self.state['names'].keys()

    def nodesxml(self):
        """ return nodes in xml format """
        result = "<xml>\n"
        gotit = False
        for name, node in self.nodes.iteritems():
            gotit = True
            result += "    <coll>\n"
            result += "        <url>%s</url>\n" % node.url
            result += "        <id>%s</id>\n" % node.id
            result += "        <name>%s</name>\n" % node.name
            result += "    </coll>\n"
        if gotit:
            result += "</xml>"
            return result
        return ""

colls = Collective()

def init():
    """ init the collective plugin """
    if not coll['enable']:
        return 1
    time.sleep(5)
    try:
        colls.boot()
    except Exception, ex:
        rlog(10, 'collective', 'error booting: %s' % str(ex))
    colls.start()
    rlog(10, 'collective', 'total of %s nodes' % size())
    return 1

def shutdown():
    """ shutdown the collective plugin """
    return 1

def size():
    """ return number of active collective nodes """
    return colls.size()

def handle_collping(bot, ievent):
    """ do a ping on a collective node """
    if not coll['enable']:
        ievent.reply('collective is not enabled')
        return
    try:
        name = ievent.args[0]
    except IndexError:
        ievent.missing('<name>')
        return
    id = colls.state['names'][ievent.rest]
    node = colls.get(id)
    result = node.ping()
    if 'pong' in result:
        ievent.reply('%s is alive' % name)
    else:
        ievent.reply('%s is not alive' % name)

cmnds.add('coll-ping', handle_collping, 'OPER')
examples.add('coll-ping', 'ping a collective node', 'coll-ping gozerbot')

def handle_colllist(bot, ievent):
    """ coll-list .. list all nodes in cache """
    if not coll['enable']:
        ievent.reply('collective is not enabled')
        return
    ievent.reply("collective nodes: ", colls.list(), dot=' \002||\002 ')

cmnds.add('coll-list', handle_colllist, 'OPER')
examples.add('coll-list', 'list nodes cache', 'coll-list')

def handle_collenable(bot, ievent):
    """ coll-enable .. enable the collective """
    ievent.reply('enabling collective')
    coll['enable'] = 1
    coll.save()
    plugins.reload('gozerplugs.plugs', 'collective')
    time.sleep(4)
    ievent.reply('done')

cmnds.add('coll-enable', handle_collenable, 'OPER')
examples.add('coll-enable', 'enable the collective', 'coll-enable')

def handle_colldisable(bot, ievent):
    """ coll-disable .. disable the collective """
    coll['enable'] = 0
    coll.save()
    plugins.reload('gozerplugs.plugs', 'collective')
    ievent.reply('collective disabled')

cmnds.add('coll-disable', handle_colldisable, 'OPER')
examples.add('coll-disable', 'disable the collective', 'coll-disable')

def handle_collsync(bot, ievent):
    """ coll-sync <node> .. sync nodes cache with node """ 
    if not coll['enable']:
        ievent.reply('collective is not enabled')
        return
    try:
        url = ievent.args[0]
    except IndexError:
        ievent.missing('<url>')
        return
    try:
        result = colls.sync(url)
    except IOError, ex:
        ievent.reply('ERROR: %s' % str(ex))
        return
    ievent.reply('%s nodes added' % result)

cmnds.add('coll-sync', handle_collsync, 'OPER', allowqueue=False)
examples.add('coll-sync', 'coll-sync <url> .. sync with provided node', \
'coll-sync http://gozerbot.org:8088')

def handle_coll(bot, ievent):
    """ coll <cmnd> .. execute <cmnd> on nodes """
    if not coll['enable']:
        ievent.reply('collective is not enabled')
        return
    if not ievent.rest:
        ievent.missing('<command>')
        return
    starttime = time.time()
    command = ievent.rest
    waitres = re.search(waitre, command)
    if waitres:
        wait = waitres.group(1)
        try:
            wait = int(wait)
        except ValueError:
            ievent.reply('wait needs to be an integer')
            return
        command = re.sub(waitre, '', command)
    else:
        wait = 5
    result = colls.cmndall(command + ' chan %s' % ievent.channel, wait)
    if result:
        reply = '%s out of %s (%s) => ' % (len(result), size(), time.time() \
- starttime)
        ievent.reply(reply, result, dot='\002||\002')
    else:
        ievent.reply('no results found')

cmnds.add('coll', handle_coll, ['USER', 'WEB'], allowqueue=False)
examples.add('coll', 'coll <cmnd> .. execute command in the collective', \
'coll lq')

def handle_collexec(bot, ievent):
    """ coll <nodename> <cmnd> .. execute <cmnd> on node """
    if not coll['enable']:
        ievent.reply('collective is not enabled')
        return
    try:
        (name, command) = ievent.rest.split(' ', 1)
    except ValueError:
        ievent.missing('<nodename> <command>')
        return
    node = colls.byname(name)
    if not node:
        ievent.reply('no node %s found' % name)
        return
    waitres = re.search(waitre, command)
    if waitres:
        wait = waitres.group(1)
        try:
            wait = int(wait)
        except ValueError:
            ievent.reply('wait needs to be an integer')
            return
        command = re.sub(waitre, '', command)
    else:
        wait = 5
    starttime = time.time()
    queue = Queue.Queue()
    command = command + ' chan %s' % ievent.channel
    start_new_thread(node.cmnd, (command, queue))
    res = waitforqueue(queue, wait+2, maxitems=1) 
    result = []
    for i in res:
        result.append(i[1])
    if result:
        ievent.reply("(%s) => [%s] " % (time.time() - starttime, name), \
result, dot=True)
    else:
        ievent.reply('no results found')

cmnds.add('coll-exec', handle_collexec, ['USER', 'WEB'], allowqueue=False)
examples.add('coll-exec', 'coll <nodename> <cmnd> .. execute command in \
the collective', 'coll-exec gozerbot lq')

def handle_colladdnode(bot, ievent):
    """ coll-addnode <name> <host:port> .. add node to cache """
    if not coll['enable']:
        ievent.reply('collective is not enabled')
        return
    try:
        (name, url) = ievent.args
    except ValueError:
        ievent.missing('<name> <url>')
        return
    colls.add(name, url)
    colls.persist(name, url)
    ievent.reply('%s added' % name)

cmnds.add('coll-add', handle_colladdnode, 'OPER')
examples.add('coll-add', 'coll-add <name> <url> .. add a node to cache and \
persist it', 'coll-add gozerbot http://gozerbot.org:8088')

def handle_collgetnode(bot, ievent):
    """ coll-getnode .. show node of <name>  """
    if not coll['enable']:
        ievent.reply('collective is not enabled')
        return
    try:
        name = ievent.args[0]
    except IndexError:
        ievent.missing('<name>')
        return
    try:
        node = colls.byname(name)
    except KeyError:
        ievent.reply('no such node')
    ievent.reply(str(node))
 
cmnds.add('coll-getnode', handle_collgetnode, 'OPER')
examples.add('coll-getnode', 'coll-getnode <name> .. get node of <name>', \
'coll-getnode gozerbot')

def handle_collnames(bot, ievent):
    """ coll-names .. show names with nodes in cache """
    if not coll['enable']:
        ievent.reply('collective is not enabled')
        return
    ievent.reply("collective node names: ", colls.names(), dot=True)
 
cmnds.add('coll-names', handle_collnames, 'OPER')
examples.add('coll-names', 'show all node names', 'coll-names')

def handle_collboot(bot, ievent):
    """ boot the collective node cache """
    if not coll['enable']:
        ievent.reply('collective is not enabled')
        return
    try:
        url = ievent.args[0]
    except IndexError:
        url = 'http://gozerbot.org:8088'
    try:
        bootnr = colls.boot(url)
    except IOError, ex:
        ievent.reply('ERROR: %s' % str(ex))
        return
    if bootnr:
        ievent.reply('collective added %s nodes' % bootnr)
    else:
        ievent.reply("no new nodes added from %s" % url)
 
cmnds.add('coll-boot', handle_collboot, 'OPER')
examples.add('coll-boot', 'sync collective list with provided host', \
'1) coll-boot 2) coll-boot http://localhost:8888')

def handle_collfullboot(bot, ievent):
    """ coll-fullboot .. boot from all nodes in cache """
    if not coll['enable']:
        ievent.reply('collective is not enabled')
        return
    try:
        teller = colls.fullboot()
    except IOError, ex:
        ievent.reply('ERROR: %s' % str(ex))
        return
    ievent.reply('%s nodes checked .. current %s nodes in list' % (teller, \
size()))
 
cmnds.add('coll-fullboot', handle_collfullboot, 'OPER')
examples.add('coll-fullboot', 'do a boot on every node in the collective \
list', 'coll-fullboot')

def handle_collremove(bot, ievent):
    if not coll['enable']:
        ievent.reply('collective is not enabled')
        return
    if not ievent.rest:
        ievent.missing('<name>')
        return
    got = False
    try:
        id = colls.state['names'][ievent.rest]
        if id:
            colls.unpersist(id)
            colls.remove(id)
            got = True
    except Exception, ex:
        ievent.reply('error removing %s: %s' % (ievent.rest, str(ex)))
        return
    if got:
        ievent.reply('%s node removed' % ievent.rest)
    else:
        ievent.reply('error removing %s node' % ievent.rest)

cmnds.add('coll-remove', handle_collremove, 'OPER')
examples.add('coll-remove', 'remove node with <name> from collective' , \
'coll-remove gozerbot')

def handle_colljoin(bot, ievent):
    if not coll['enable']:
        ievent.reply('collective is not enabled')
        return
    if not ievent.rest:
        ievent.missing('<name>')
        return
    try:
        id = colls.state['names'][ievent.rest]
        node = colls.get(id)
        result = colls.join(node.url)
    except Exception, ex:
        ievent.reply('error joining %s: %s' % (ievent.rest, str(ex)))
        return
    ievent.reply(result)

cmnds.add('coll-join', handle_colljoin, 'OPER')
examples.add('coll-join', 'join node with <name>' , 'coll-join gozerbot')