# Copyright (c) 2001 LOGILAB S.A. (Paris, FRANCE).
# http://www.logilab.fr/ -- mailto:contact@logilab.fr
#
# This program is free software; you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free Software
# Foundation; either version 2 of the License, or (at your option) any later
# version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along with
# this program; if not, write to the Free Software Foundation, Inc.,
# 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.

"""Guess_language tries to guess the language of an HTML page using a
simple scoring technique (coefficients were chosed by hand, but we may end
up using a learning algorithm later on).

For now, it can recognize 6 languages: italian, spanish, german, portuguese,
english and french.
"""

__revision__ = "$Id: guess_language.py,v 1.2 2001/11/22 16:16:33 syt Exp $"


from narval.lib import *

source_vector = load_extension_module('classify','source_vector')
myhtmllib = load_extension_module('html','myhtmllib')

import string
import re

# data #########################################################################

LANGUAGE_LIST = ['en','fr','it','pt','es','de']

DICT = {

    'en':{ # -- ENGLISH
    # words
    re.compile('^yes$'): 1, re.compile('^the$'):1, re.compile('^you$'):1,
    re.compile('^have$'):1, re.compile('^is$'): 1, re.compile('^and$'):1,
    re.compile('^by$'):  1, re.compile('^for$'):1, re.compile('^who$'):1,
    re.compile('^when$'):1, re.compile('^why$'):1, re.compile('^in$'):0.2,
    re.compile('^to$'):0.2,
    # end of words
    re.compile('.+y$'):0.3, re.compile('.+ght$'):0.2, re.compile('.+ing$'):0.1
    },

    'it':{ # -- ITALIAN
    # words 
    re.compile('^gli$'):  1, re.compile('^di$'):  1, re.compile('^nel$'):1,
    re.compile('^ed$'):   1, re.compile('^per$'): 1, re.compile('^i$'):1,
    re.compile('^della$'):1, re.compile('^sono$'):1, re.compile('^dell$'):2,
    re.compile('^e$'):  0.3, re.compile('^con$'):0.2,re.compile('^pi\371$'):0.7,
    # end of words
    re.compile('.+\340$'): 0.2, re.compile('.+zione$'):0.6,
    re.compile('.+zioni$'):0.6, re.compile('.+zie$'):  0.3,
    re.compile('.+zia$'):  0.3, re.compile('.+ieri$'): 0.1,
    re.compile('.+i$'):   0.05
    },
    
    'de':{ # -- GERMAN
    # words
    re.compile('^der$'):1,  re.compile('^das$'):  1, re.compile('^ich$'):1,
    re.compile('^und$'):1,  re.compile('^nicht$'):1, re.compile('^f\374r$'):1,
    re.compile('^von$'):0.3,re.compile('^die$'):0.1,
    # end of words 
    re.compile('.+en$'):0.1,re.compile('.+ag$'):0.2,
    # patterns
    re.compile('.*isch.*'):0.2, re.compile('.*cht.*'):0.2,
    re.compile('.*wert.*'):0.5, re.compile('.*ung.*'):0.2,
    re.compile('.*k[aiou\366].*'):0.2
    },
    
    'es':{ # -- SPANISH
    # words
    re.compile('^del$'):1,re.compile('^los$'):1,re.compile('^las$'):1,
    re.compile('^este$'):1,re.compile('^el$'):1,re.compile('^por$'):1,
    re.compile('^y$'):0.2,
    # end of words
    re.compile('.+ci\363n$'):0.2,re.compile('.+ad$'):0.2,re.compile('.+ar$'):0.2
    },

    'pt':{ # -- PORTUGUESE
    # words
    re.compile('^do$'):1,re.compile('^seu$'):1,re.compile('^seus$'):1,
    #patterns
    re.compile('^.*\343o.*$'):1,re.compile('^.*\365e.*$'):1,re.compile('^.*\365a.*$'):1,
    #end of words
    re.compile('^.+\352$'):1
    },
    
    'fr':{ # -- FRENCH
    # words
    re.compile('^le$'): 1, re.compile('^des$'): 1, re.compile('^\340$'):1,
    re.compile('^et$'): 1, re.compile('^pour$'):1, re.compile('^par$'): 1,
    re.compile('^est$'):1, re.compile('^vous$'):1, re.compile('^nous$'):1,
    re.compile('^ils$'):1, re.compile('^qui$'): 1, re.compile('^quoi$'):1,
    re.compile('^de$'):0.3,re.compile('^il$'):0.2, re.compile('^l$'): 0.3,
    # patterns
    re.compile('^.+\350.*$'):0.1,re.compile('^.*\351.*\351.*$'):0.8
    }

    } # -- end of DICT
    

# SPECIAL_CHARACTERS
SCHARS   = {
    'en':{},
    'it':{},
    'de':{'\337':1,'\354':0.5,'\344':0.5,'\366':0.5,'\374':0.5},
    'es':{'\361':1,'\355':0.2,'\363':0.5},
    'pt':{'\343':1,'\365':1},
    'fr':{'\350':0.1,'\351':0.1}
    }


# CONDITIONS
U_RATIO   = {'en':5,'it':5,'de':5,'es':5,'pt':5,'fr':5}
U_NBWORDS = {'en':10,'it':10,'de':10,'es':10,'pt':7,'fr':10}
U_SCHARS  = {'de':5,'es':5,'pt':4,'fr':6}

COUPLED_CONDITIONS = {
                      'de':[3,4],
                      'es':[3,3],
                      'pt':[3,2],
                      }

# functions ####################################################################


# This function is called when we're not sure of the text's language
# The result will be the value returned by max(dict).

def max(dict):
    """
    for each key of the dictionnary passed in argument, compute
    (dict[key][0]*(dict[key][1]+1)),
    and return the key corresponding to the highest value found.
    """
    r = None
    max = 0
    for key in dict.keys() :
        if r == None :
            r = key
            max = dict[key][0]*(dict[key][1]+1)
        else:
            x = dict[key][0]
            y = dict[key][1] + 1
            if x*y > max:
                r= key
                max = x*y
    return r


def test_lang(lang,word_list,enough = 1):
    """
    Tests if the list of words matches a given language. The general idea is,
    for each word in a text, to try to find if this word match any singularity
    (special word, special character, etc.) of the known languages.

    Arguments:
      lang       the language ('en', 'fr', 'de', ...)
      word_list  a list of couples (word, # occurences in the text)
      enough     less than 20 different words is not enough

    Return value:
      if we're sure of the text's language : return 1
      else : return a couple representing the number of special patterns and of
      special characters found
    """
    
    nb_words, special_char, total_nb_words = 0, 0, 0

    for word in word_list.keys():
        nb_occurs = word_list[word]
        total_nb_words = total_nb_words + nb_occurs
        
        # for the specified language, tests if one of the predefined pattern
        # matches the word and then increments consequently nb_words
        try:
            regexp_dict = DICT[lang]
            for pattern in regexp_dict.keys():
                if pattern.match(word):
                    nb_words = nb_words + regexp_dict[pattern]*nb_occurs
                    break
        except KeyError:
            print 'No such language as', lang
            pass

        # same thing but with special characters instead of patterns
        try:
            schars_dict = SCHARS[lang]
            for spec_char in schars_dict.keys():
                try:
                    string.index(word,spec_char)
                    special_char = special_char+schars_dict[spec_char]*nb_occurs
                except ValueError:
                    pass
        except KeyError:
            pass 
    
    # if not enough words in the text => decide later
    if not enough:
        if nb_words == 1:
            nb_words = 0.9
        #print "*"*60
        return nb_words,special_char

    # % 
    ratio = nb_words*100/total_nb_words

    
    try:          
        c_ratio = COUPLED_CONDITIONS[lang][0]
        c_schar = COUPLED_CONDITIONS[lang][1]
        # if this test works, we're sure of the language => return 1
        if ratio > c_ratio and special_char > c_schar:
            return 1,special_char
    except KeyError:
        pass

     # if this test works, we're sure of the language => return 1
    if ratio > U_RATIO[lang]:
        return 1,special_char

    try:
        # if this test works, we're sure of the language => return 1
        if special_char > U_SCHARS[lang]:
            return 1,special_char
    except KeyError:
        pass
    
    # Here, we'll decide later
    if nb_words == 1:
        nb_words = 0.9

    return nb_words,special_char
    
def identify_language(list_vector):
    res_dict={}
    enough = 1

    # warn if not enough different words in the text
    if len(list_vector) < 20:
        if not list_vector:
            print "No words in this document !"
            return 'en'
        enough = 0
        print 'Warning : only ',len(list_vector),' different words !'

    # try to guess the language
    for language in LANGUAGE_LIST:
        nb_words, schars = test_lang(language,list_vector,enough)
        res_dict[language] = [nb_words,schars]

        if not enough:
            continue
        if nb_words == 1:
            return language
    
    return max(res_dict)
           
def identify_language_from_url(url):
    """
    This function takes an url, and returns the language of
    the url text ('en','de', ...)
    """
    # retrieve the text from the url
    data = myhtmllib.parse_html_url(url)
    tsv = source_vector.tranf_source_vector(use_engl_dico=0,nb_mini_letters=1)
    list_vector = tsv.do(data, debug=0, only_dico=1)
    return identify_language(list_vector)

def identify_language_from_buffer(buf):
    """
    This function takes a buffer, and returns the language of
    this buffer's text ('en','de', ...)
    """
    from cStringIO import StringIO
    from formatter import AbstractFormatter, DumbWriter
    
    data = StringIO()
    form = AbstractFormatter(DumbWriter(data))
    p = myhtmllib.MyHTMLParser(form)
    p.feed(buf)
    p.close()
    tsv = source_vector.tranf_source_vector(use_engl_dico=0,nb_mini_letters=1)
    list_vector = tsv.do(data, debug=0, only_dico=1)
    return identify_language(list_vector)
    

# test #########################################################################
#  def browse_directory(dir_path,fileList):
#      """
#      This function opens a directory, then for all its files, reads each line
#      (corresponding to an url) and guess its language
#      """
#      res_file = open("/home/adim/proxy/python/results","w")
#      for file in fileList :
#          lang = file
#          filePath=dir_path+'/'+file
#          f = open(filePath)
#          urlList = f.readlines()
#          print "*"*20+' '+lang+' '+"*"*20
#          res_file.write("*"*20+' '+lang+' '+"*"*20)
#          res_file.write("\n")
#          for url in urlList:
#              try:
#                  print "url : ",url,
#                  flang = identify_language_from_url(url)
#                  print 'expected "',lang,'", found "',flang,'"'   
#              except:
#                  pass
#          res_file.flush()
#      res_file.close()
#  def browse_file(filePath):
#      """
#      This function opens a file, and then for each line of the file (corresponding
#      to an url), guess the url text language
#      """
#      try:
#          f = open(filePath)
#          urlList = f.readlines()
#          parsedPath=string.split(path,'/')
#          lang=parsedPath[-1]       
#          for url in urlList:
#              print "url : ",url,
#              try:
#                  flang = identify_language_from_url(url)
#                  print 'expected "',lang,'", found "',flang,'"'
#              except:
#                  import traceback
#                  traceback.print_exc()
#                  pass
#      except:
#          print sys.exc_info()[0], 'open ',filePath,' failed'
#
# main #########################################################################
if __name__ == '__main__':
    import sys
    import os
    args = sys.argv[1:]
        
    if args :
        for path in args :
            try:
                lang = identify_language_from_url(path)
                print path,"-->",lang
            except Exception, e:
                import traceback
                traceback.print_exc()
                pass
    else:
        print "Usage: guess_language.py URL ..."
        print "As in guess_language http://www.logilab.fr/ "+\
              "file:///home/user/file.html"
