# Copyright (c) 2000 LOGILAB S.A. (Paris, FRANCE).
# http://www.logilab.fr/ -- mailto:contact@logilab.fr
#
# This program is free software; you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free Software
# Foundation; either version 2 of the License, or (at your option) any later
# version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along with
# this program; if not, write to the Free Software Foundation, Inc.,
# 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
__revision__ = "$Id: Basic.py,v 1.67 2002/02/22 17:30:11 alf Exp $"

import copy, os
import urllib, urlparse, string
import traceback
import re, string
from narval.lib import AL_NS, NO_NS, get_encoding, normalize_url, \
     select_one, select_many, SelectException, write_domtree_to_file, \
     read_domtree_from_uri, read_domtree_from_string, load_extension_module,\
     error, PrettyPrint, false, domtree_to_string
from narval.core.ALAbstraction  import Document

MOD_XML = """<module xmlns:al='%s'>""" % AL_NS
# FIXME: handle proxy correctly

# No OPeration #################################################################
def NOP_f(args, doc) :
    return {}

MOD_XML = MOD_XML + \
          """<al:action name='NOP' func='NOP_f'>
           <al:description lang='en'>the very usefull NO oPeration !</al:description>
          </al:action>
          """

# Execute a command ############################################################
def command_f(args, doc):
    command = args['input'].firstChild.data
    o = {}
    if os.system(command):
        o['error'] = [error(doc, 'Error while execute command "%s"'%command)]
    return o

MOD_XML = MOD_XML + \
          """<al:action name='command' func='command_f'>
          <al:description lang='en'>execute a system command</al:description>
          <al:input id='input' use='yes'><al:match>command/text()</al:match>
           <al:description lang='en'>the command to execute</al:description></al:input>
          </al:action>
          """



# Read a file ##################################################################
def read_raw_file_f(args, doc):
    # create document with root outputs
    output = {}
    url = args['to-read'].firstChild.data

    try:
        # read data
        f = urllib.urlopen(normalize_url(url)[0])
        # remove unacceptable \0 character from read data -- happens with yahoo
        tempdata = f.read()
        data = re.sub('\x00','',tempdata)
        f.close()
        
        # dom accepts only utf-8 or unicode and narval wants unicode
        data = unicode(data, get_encoding())
        # document
        documentNode = doc.createElementNS(NO_NS, 'raw-file')
        documentNode.setAttributeNS(NO_NS,'type','read')
        # path
        urlNode = doc.createElementNS(NO_NS, 'url')
        urlNode.appendChild(doc.createTextNode(url))
        documentNode.appendChild(urlNode)
        # content
        dataNode = doc.createElementNS(NO_NS, 'data')
        dataNode.appendChild(doc.createTextNode(data))
        documentNode.appendChild(dataNode)
        output['output'] = documentNode      
    except IOError:
        output['error'] = [error(doc, 'No such url %s'%url)]
    
    return output

MOD_XML = MOD_XML + \
          """<al:action name='read_file' func='read_raw_file_f'>
          <al:description lang='en'>read a raw file</al:description>
          <al:input id='to-read'><al:match>url/text()</al:match>
          <al:description lang='en'>url locating the file to read</al:description></al:input>
          <al:output id='output'><al:match>raw-file/url/text()</al:match>
                  <al:match>raw-file/data/text()</al:match>
          <al:description lang='en'>xml element containing the raw file in its data child</al:description>
          </al:output>
          </al:action>
          """

# Read an XML file #############################################################

def read_xml_file_f(args, doc):
    outputs = {}
    url = args['to-read'].firstChild.data
##     PrettyPrint(args['to-read'])
##     print 'URL: ',url
    proxy = args['proxy']
    if proxy:
        from Http import export_http_proxy
        export_http_proxy(proxy)
    try:
        # read data
        # FIXME: rajouter Proxies et URLOpener
        try:
            docnode = read_domtree_from_uri(url).documentElement
            outputs['output'] = docnode
        except Exception,e:
            import traceback
            traceback.print_exc()
            f = urllib.urlopen(normalize_url(url)[0])
            data = f.read()
            f.close()
            # try correcting stuff
            if data[:5] == '<?xml':
                import string,re
                if string.find(data[:80],'encoding')==-1:
                    p = string.find(data,'?>')
                    data = '%s encoding="utf-8" %s'% (data[:p], data[p:])
                else:
                    raise e
            else :
                data = '<?xml version="1.0" encoding="%s"?>\n%s'%(
                    data, get_encoding())
            docnode = read_domtree_from_string(data).documentElement
            outputs['output'] = docnode
    except IOError, e:
        import traceback
        traceback.print_exc()
        outputs['error'] = [error(doc, 'No such url %s'%url)]
    return outputs

MOD_XML = MOD_XML + """
<al:action name='read_xml_file' func='read_xml_file_f'>
<al:description lang='en'>read a valid xml file</al:description>
<al:input id='to-read'><al:match>url/text()</al:match>
<al:description lang='en'>url locating the file to read</al:description></al:input>
<al:input id='proxy' optional="yes"><al:match>proxy[@type="http"]</al:match>
<al:description lang="en">proxy element, for page requests</al:description></al:input>
<al:output id='output'>
  <al:description lang='en'>
xml element which represents the readen file (it's root element depends on the
file you have just read!)
  </al:description>
</al:output>
</al:action>
"""

# Write a file #################################################################

def write_raw_file_f(args, doc) :
    rawf = args['input']
    url = select_one(rawf,'url/text()').data
    url, url_tuple = normalize_url(url)
    data = select_one(rawf,'data').firstChild.data
    o = {}
    if url_tuple[0] == 'file' :
        fname = _path_from_url(url_tuple[2])
        f = open(fname,'w')
        f.write(data)
        f.close()
    else :
        o['error'] = [error(doc, 'No such url %s'%url)]
    return o

MOD_XML = MOD_XML + \
          """<al:action name='write_file' func='write_raw_file_f'>
           <al:description lang='en'>write a raw file</al:description>
          <al:input id='input'>
          <al:match>raw-file/url/text()</al:match>
          <al:match>raw-file/data/text()</al:match>
           <al:description lang='en'>url locating the file to write</al:description>
           </al:input>
          </al:action>
          """

# Write an XML file ############################################################

def write_xml_file_f(args, doc) :
    # get the url where the file will be written
    url = args['url'].firstChild.data
    url, url_tuple = normalize_url(url)
    # get the data to be written in the file
    data = args["to-write"]

    # get the encoding of the file (optional, default is UTF-8)
    encoding = args['encoding']
    if encoding is None:
        encoding = get_encoding()
    else:
        encoding = encoding.firstChild.data
    o = {}
#     # get the doctype of the file (optional)
#     try:
#         systemId = select_one(args,'xml-file/doctype/systemid/text()').data
#     except SelectException:
#         systemId = ""
#     try:
#         publicId = select_one(args,'xml-file/doctype/publicid/text()').data
#     except SelectException:
#         publicId = ""
#     if systemId=="":
#         # publicId can't be defined alone (cf. XML Spec)
#         publicId = ""
#     # Builds a doctype for the file
#     docType = implementation.createDocumentType(data.localName,
#                                                 publicId, systemId)
#     # Builds a document for the file
#     document = implementation.createDocument('', '', docType)
#     # Adds the data to be saved as child of this document
#     document.appendChild(document.importNode(data,1))
    # Opens the file and saves the document
    if url_tuple[0] == 'file' :
        fname = _path_from_url(url_tuple[2])
        write_domtree_to_file(data, fname, encoding)
    else :
        o['error'] = [error(doc, 'Can\'t write file at url %s'%url)]
    return o

MOD_XML = MOD_XML + """<al:action name='write_xml_file' func='write_xml_file_f'>
 <al:description lang='en'>write an xml file</al:description>
 <al:input id='url'>
   <al:match>url[text()]</al:match>
   <al:description lang='en'>url locating the file to write</al:description>
 </al:input>
 <al:input id='encoding' optional='yes'>
   <al:match>encoding/text()</al:match>
   <al:description lang='en'>encoding to use in the file</al:description>
 </al:input>
 <al:input id='to-write'>
  <al:description lang='en'>xml element which represents the file to write</al:description>
 </al:input>
</al:action>"""

# Write an HTML file ###########################################################

def write_html_file_f(args, doc) :
    from xml.dom.ext.XHtml2HtmlPrinter import XHtml2HtmlPrintVisitor
    from xml.dom.ext.Printer import PrintWalker
    filenode = args['input']
    # get the url where the file will be written
    url = select_one(filenode,'url/text()').data
    url, url_tuple = normalize_url(url)
    # get the data to be written in the file
    dataNodes = select_many(filenode,'data/*')
    # get the encoding of the file (optional, default is UTF-8)
    try:
        encoding = select_one(filenode,'encoding/text()').data
    except SelectException:
        encoding = get_encoding()
    o = {}
    # Builds a doctype for the file (neither systemId nor publicId is
    # necessary, the printer will insert default ones)
    ##docType = implementation.createDocumentType('xhtml','', '')
    
    # Opens the file and saves the document
    if url_tuple[0] == 'file' :
        fname = _path_from_url(url_tuple[2])
        f = open(fname,'w')
        visitor = XHtml2HtmlPrintVisitor(f, encoding, '    ')
        for dataNode in dataNodes :
            # Builds a document,
            document = Document()
            document.appendChild(document.createElementNS(NO_NS, 'xhtml'))
            # then merge the data.
            document.appendChild(document.importNode(dataNode,1))
            PrintWalker(visitor, dataNode).run()
        f.write('\n')
        f.close()
    else :
        o['error'] = [error(doc, 'Can\'t write file at url %s'%url)]
    return o

MOD_XML = MOD_XML + \
          """<al:action name='write_html_file' func='write_html_file_f'>
           <al:description lang='en'>write an html file</al:description>
          <al:input id='input'><al:match>html-file/url/text()</al:match>
          <al:match>html-file/data/*</al:match>
           <al:description lang='en'>html-file node contain url locating the file to write
           and the data to write in the file</al:description></al:input>
          </al:action>
          """

# Append text to a file ########################################################
def append_text_to_file_f(args, doc) :
    o = {}
    url = args['url'].firstChild.data
    text_val = args['input'].firstChild.data
    url, url_tuple = normalize_url(url)
    if url_tuple[0] == 'file' :
        fname = _path_from_url(url_tuple[2])
        f = open(fname, 'a')
        f.write('%s\n' % text_val)
        f.close()
    else:
        o['error'] = [error(doc, 'Can\'t write file at url %s'%url)]
    return o

MOD_XML = MOD_XML + \
          """<al:action name='append_text_to_file' func='append_text_to_file_f'>
           <al:description lang='en'>append some text to a file</al:description>
          <al:input id='url'><al:match>url</al:match>
           <al:description lang='en'>url locating the file to write</al:description></al:input>
          <al:input id='input'><al:match>text</al:match>
           <al:description lang='en'>text to append</al:description></al:input>
          </al:action>
          """

# Search a match to regular expressions in an element ##########################
REGEXER_ERR_MSG = """Could not find string to match, check that you do
<regexp-set match='foo/bar'><rule/></regexp-match> with
<baz><foo><bar>your text string</bar></foo><dotcom/></baz> and mind
the baz/foo/bar and foo/bar difference."""

def regexer_f(args, doc) :
    from re import search,IGNORECASE
    # init
    regexp_set = args['input']
    regexp_target = args["regexp-target"]
    o = {}
    regexp_result_node = false

    # Let's get the string to test...
    try:
        path = regexp_set.getAttributeNS(NO_NS, 'match')
        st = select_one(regexp_target,path).data
    except SelectException:
#        error = doc.createElementNS(AL_NS,'al:error')
#        error.appendChild(doc.createTextNode(REGEXER_ERR_MSG))
#        o['error'] = [error]
        o['error'] = error(doc, REGEXER_ERR_MSG)
        return o

    # Apply each rule until we find a match.
    for rule_node in select_many(regexp_set,'rule') :
        regexp = rule_node.firstChild.data
        if rule_node.getAttributeNS(NO_NS, 'ignore-case') == 'yes' :
            m = re.compile(regexp, re.I).match(st)
        else :
            m = re.compile(regexp).match(st)
        if m :
            regexp_result_node = doc.createElementNS(NO_NS, 'regexp-result')
            regexp_result_node.setAttributeNS('','rule-id',
                                              rule_node.getAttributeNS(NO_NS, 'id'))
            o['output'] = regexp_result_node
            
            for name,value in m.groupdict().items() :
                group_node = doc.createElementNS(NO_NS, 'group')
                group_node.setAttributeNS(NO_NS, 'name',name)
                value_node = doc.createTextNode(value)
                group_node.appendChild(value_node)
                regexp_result_node.appendChild(group_node)
            break
    # not match found: error
    if not regexp_result_node :
        error_node = error(doc, 'No rule matched.')
        map(error_node.appendChild, args.values())
        o['error'] = [error_node]
    return o

MOD_XML = MOD_XML + \
          """<al:action name='regexer' func='regexer_f'>
           <al:description lang='en'>regular expression matcher</al:description>
          <al:input id='input'><al:match>regexp-set[@match and rule/text()]</al:match>
           <al:description lang='en'>regular expression</al:description></al:input>
          <al:input id='regexp-target'>
           <al:description lang='en'>string to process</al:description></al:input>
          <al:output id='output'><al:match>regexp-result</al:match>
           <al:description lang='en'>result structure</al:description></al:output>
          </al:action>
          """
# Send a message ###############################################################

def send_msg_f(args, doc) :
    outputs = {}
    msg = args['message']
    #print 'Sending message',select_one(msg,'@eid/text()').data, \
    #      select_one(msg,'header/to/text()').data
    msg.removeAttributeNS(AL_NS,'eid')
    #msg.removeAttributeNS(AL_NS,'from_plan')
    #msg.removeAttributeNS(AL_NS,'from_step')
    # FIXME : many listen-on nodes are authorized, so which one to take?
    listen = args['narval'][0]
    headers = select_one(msg,'header')
    from_host = listen.getAttributeNS(NO_NS,'host')
    from_port = listen.getAttributeNS(NO_NS,'port')
    host, port = select_one(headers,'to/text()').data.split(':')
#    timestamp = select_one(msg,'@timestamp/text()').data
    timestamp = msg.getAttributeNS(NO_NS,'timestamp')
    msg.removeAttributeNS(NO_NS,'timestamp')
    from_node = msg.ownerDocument.createElementNS(AL_NS, 'al:from')
    time_node = msg.ownerDocument.createElementNS(AL_NS, 'al:timestamp')
    headers.appendChild(from_node)
    headers.appendChild(time_node)
    from_node.appendChild(doc.createTextNode(from_host+':'+from_port))
    time_node.appendChild(doc.createTextNode(timestamp))
    from narval.communication import RPCFactory
    try:
        print "HOST=", host, port
        uri = RPCFactory.encode_uri('xmlrpc', host, port)
        other = RPCFactory.create_proxy(uri)
        other.postMessage(domtree_to_string(msg))
    except:
        err = error(doc, 'unreachable host')
        err.setAttributeNS(NO_NS,'address',str(host+':'+port))
        outputs['error'] = err
        raise
    return outputs

MOD_XML = MOD_XML + """<al:action name='send_msg' func='send_msg_f'>
<al:description lang='en'>send a message to another Narval, need use of xmlrpc</al:description>
 <al:input id='message' use='yes'>
  <al:match>message/@type='outgoing'</al:match>
  <al:description lang='en'>message to send</al:description>
 </al:input>
 <al:input id='narval' list='yes'>
  <al:match>al:listen-on</al:match>
  <al:description lang='en'>structure which describes remote Narval</al:description>
 </al:input>
</al:action>
"""

def download_dataunit_f(args, doc):
    # FIXME: optional myhtmllib (->word coef)
    myhtmllib  = load_extension_module('html', 'myhtmllib')
    output = {'output': []}
    list_error=[]
    im_filter = args['filter'] or 1
    encod = get_encoding()
    for s_u in args['input']:
        try:
            data_n = select_one(s_u, 'data')
        except Exception,e:
            data_n = None
            data = ''
        else:
            try:
                txt_n = select_one(data_n, 'text()')
                data = unicode(txt_n.data, encod)
                data_n.removeChild(txt_n)
            except Exception,e:
                data = ''
        url = select_one(s_u, 'url/text()').data
        # image filter
        if im_filter:
            ext_list = re.split('[.*]', url[-5:])
            end = len(ext_list)-1
            if ext_list[end] == 'jpg' or \
               ext_list[end] == 'png' or \
               ext_list[end] == 'gif':
                print 'url is a picture'
                continue
        try:
            txt_data = myhtmllib.parse_html_url(url)
            from cStringIO import StringIO
            data = StringIO('%s%s' % (txt_data.getvalue(), data))
        except :
            # 'Error, Pass this url.'
            print 'ERROR in url', url
            import traceback
            traceback.print_exc()
            list_error.append(s_u)
            continue
        if data_n is None:
            data_n = doc.createElementNS(NO_NS, 'data')
            s_u.appendChild(data_n)
        data_txt = doc.createTextNode(unicode(data.getvalue(), get_encoding()))
        data_n.appendChild(data_txt)
        #PrettyPrint(s_u)
        output['output'].append(s_u)
    # FIXME error handling in different ns ?
    return output

MOD_XML = MOD_XML + """<al:action name='download_data-unit' func='download_dataunit_f'>
<al:description lang='en'>take a data-unit element with an url child, return the data-unit
element with its data child concatened with the content downloaded from the url
</al:description>
<al:input id='input' use='yes' list='yes'>
  <al:match>data-unit[url/text()]</al:match>
  <al:description lang='en'>
data-unit element with data to download from its url child
  </al:description>
</al:input>
<al:input id='filter' optional='yes'>
  <al:match>image-filter</al:match>
  <al:description lang='en'>
filters images if this element is found
  </al:description>
</al:input>
<al:output id='output' list='yes'>
  <al:match>data-unit</al:match>
  <al:description lang='en'>completed data-unit</al:description>
</al:output>
</al:action>
"""

MOD_XML = MOD_XML + "</module>"


# Private #####################################################################
def _path_from_url(url):
    if url[0]=='/' and url[2] == ':':
        return url[1:]
    else:
        return url


# Test #########################################################################
if __name__ == "__main__" :
    from xml.dom.ext.reader import Sax2
    print MOD_XML
    doc=Sax2.FromXml(MOD_XML)
    print doc.documentElement
