""" test unit for infopal extensions.
    define a class for test files:
     - source_vector.py
     - process_vectors.py
     - sort_vectors_mc.py
     - myhtmllib.py
     - guess_language.py
"""
        
# Sylvain Thenault - 04/07/2001
from cStringIO import StringIO
from copy import deepcopy    
import unittest
import misc


#####        
from source_vector import *
class SourceVectorKnownValues(unittest.TestCase):
    """
    This class check that source_vector functions give known result
    with known input
    """
    
    DATA = StringIO("""
    This is a test string to ensure that the count_word_occurs
    function extract the right number of words occurs from this
    string (text test string extract) (this is from the Narval
    package).
    """)
    # those variables may change if the "DATA" string change
    HKNOWN_VALUES = {
        'string':   3,
        'test':     2,
        'extract':  2,
        'occurs':   2,
        'text':     1,
        'function': 1,
        'ensure':   1,
        'word':     1,
        'words':    1,
        'count':    1,
        'right':    1,
        'number':   1,
        'narval':   1,
        'package':  1
        }
    LKNOWN_VALUES = (
        ('string',   3),
        ('test',     2),
        ('extract',  2),
        ('occurs',   2), 
        ('text',     1),
        ('function', 1), 
        ('ensure',   1), 
        ('word',     1), 
        ('words',    1), 
        ('count',    1), 
        ('right',    1),
        ('number',   1),
        ('narval',   1),
        ('package',  1)
        )
    LREPR_KNOWN_VALUES = (
        ('string',   1077),
        ('test',     718),
        ('extract',  718),
        ('occurs',   718), 
        ('text',     359),
        ('function', 359), 
        ('ensure',   359), 
        ('word',     359), 
        ('words',    359), 
        ('count',    359), 
        ('right',    359),
        ('number',   359),
        ('narval',   359),
        ('package',  359) 
        )
    NB_WORD       = 33
    NB_DIFF_WORD  = 14
    NB_MAX_OCCURS = 3
    INDICE_NUM_L  = 0
    INDICE_NUM_R  = NB_DIFF_WORD
    
    def setUp (self):
        """ called before each test from this class """
        self.tsv = tranf_source_vector()
        
    def test_cwo_known_values(self):
        """
        count_words_occurs should give known result with known input
        """

        vect = self.tsv.count_words_occurs(self.DATA)
        print vect
        for word, occurs in vect: 
            self.assert_(len(word) >= self.tsv.nb_mini_letters, \
                         'word "' + word + '" too short') 
            self.assert_(self.HKNOWN_VALUES.has_key(word), \
                         'word "' + word + '" unknown')
            self.assert_(occurs == self.HKNOWN_VALUES[word], \
                         'Wrong word occurs count for word "' + word + '"')
        self.assert_(self.tsv.nb_word == self.NB_WORD, \
                     'Wrong total word count')
        self.assert_(self.tsv.nb_diff_word == self.NB_DIFF_WORD, \
                     'Wrong diff word count')
        self.assert_(self.tsv.nb_max_occurs == self.NB_MAX_OCCURS, \
                     'Wrong max word occurs count')


    def test_scv_known_values(self):
        """
        search_cut_vector should give known result with known input
        """
        
        self.tsv.nb_max_occurs = self.NB_MAX_OCCURS
        self.tsv.nb_diff_word = self.NB_DIFF_WORD
        ind_vect = self.tsv.search_cut_vector(self.LKNOWN_VALUES)

        self.assert_(self.tsv.indice_num_r == self.INDICE_NUM_R, \
                     'Wrong right indice')
        self.assert_(self.tsv.indice_num_l == self.INDICE_NUM_L, \
                     'Wrong left indice')

        
    def test_cv_known_values(self):
        """
        cut_vector should give known result with known input
        """
        
        self.tsv.indice_num_r = self.INDICE_NUM_R
        self.tsv.indice_num_l = self.INDICE_NUM_L
        vect = self.tsv.cut_vector(self.LKNOWN_VALUES, debug=0)

        for word, norm in self.LREPR_KNOWN_VALUES:
            self.assert_(norm == vect[word], \
                         'Wrong norm for word "' + word + '"')

    def sanity_check(self):
        """
        global sanity check for do() function
        """
        vect = self.tsv.do(self.DATA, only_dico=0, debug=0)

        for word, norm in self.LREPR_KNOWN_VALUES:
            self.assert_(norm == vect[word], \
                         'Wrong norm for word "' + word + '"')
            
        vect = self.tsv.do(self.DATA, only_dico=1, debug=0)

        for word  in vect.keys():
            occurs = vect[word]
            self.assert_(len(word) <= self.tsv.nb_mini_letters, \
                         'word "' + word + '" too short')
            self.assert_(self.HKNOWN_VALUES.has_key(word), \
                          'word "' + word + '" unknown')
            self.assert_(occurs == self.HKNOWN_VALUES[word], \
                         'Wrong word occurs count for word "' + word + '"')


        
#####
from process_vectors import *
class ProcessVectorsKnownValues(unittest.TestCase):
    """
    This class check that process_vectors functions give known result
    with known input
    """
    
    LKNOWN_VALUES=[[],[]]
    HKNOWN_VALUES=[{},{}]
    HKNOWN_VALUES_N=[{},{}]
    HKNOWN_VALUES_N_CV=[{},{}]
    LKNOWN_VALUES[0] = {
        'string':   3,
        'discuss':  2,
        'test':     2,
        'extract':  2,
        'occurs':   2, 
        'text':     1,
        'function': 1, 
        'ensure':   1, 
        'word':     1, 
        'words':    1, 
        'count':    1, 
        'right':    1,
        'number':   1,
        'narval':   1,
        'package':  1
        }
    HKNOWN_VALUES[0] = {
        'discuss':  0.0924196240747,
        'extract':  0.0924196240747,
        'occurs':   0.0924196240747,
        'text':     0.0462098120373,
        'function': 0.0462098120373,
        'ensure':   0.0462098120373,
        'word':     0.0462098120373,
        'words':    0.0462098120373,
        'count':    0.0462098120373,
        'right':    0.0462098120373,
        'number':   0.0462098120373,
        'string':   0.0,
        'test':     0.0,
        'narval':   0.0,
        'package':  0.0
        }
    HKNOWN_VALUES_N[0] = {
        'discuss':  894.427191,
        'extract':  894.427191,
        'occurs':   894.427191,
        'text':     447.2135955,
        'function': 447.2135955,
        'ensure':   447.2135955,
        'word':     447.2135955,
        'words':    447.2135955,
        'count':    447.2135955,
        'right':    447.2135955,
        'number':   447.2135955,
        'string':   0.0,
        'test':     0.0,
        'narval':   0.0,
        'package':  0.0
        }
    HKNOWN_VALUES_N_CV[0] = {
        'discuss':  676.123403783,
        'extract':  676.123403783,
        'occurs':   676.123403783,
        'text':     338.061701891,
        'function': 338.061701891,
        'ensure':   338.061701891,
        'word':     338.061701891,
        'words':    338.061701891,
        'count':    338.061701891,
        'right':    338.061701891,
        'number':   338.061701891,
        'string':   1014.18510567,
        'test':     676.123403783,
        'narval':   338.061701891,
        'package':  338.061701891
        }     
    LKNOWN_VALUES[1] = {
        'string':     2,
        'horn':       2,
        'test':       1,
        'validation': 1,
        'another':    1, 
        'main':       1, 
        'narval':     1,
        'package':    1
        }
    HKNOWN_VALUES[1] = {
        'horn':       0.17328679514,
        'validation': 0.08664339757,
        'another':    0.08664339757,
        'main':       0.08664339757,
        'string':     0.0,
        'test':       0.0,
        'narval':     0.0,
        'package':    0.0
        }
    HKNOWN_VALUES_N[1] = {
        'horn':       1511.85789204,
        'validation': 755.928946018,
        'another':    755.928946018,
        'main':       755.928946018,
        'string':     0.0,
        'test':       0.0,
        'narval':     0.0,
        'package':    0.0
        }
    HKNOWN_VALUES_N_CV[1] = {
        'horn':       1069.04496765,
        'validation': 534.522483825,
        'another':    534.522483825,
        'main':       534.522483825,
        'string':     1069.04496765,
        'test':       534.522483825,
        'narval':     534.522483825,
        'package':    534.522483825
        }

    MYVECT = []
    tmpvect = []
    tmpvect.append("bookmark1")
    tmpvect.append(LKNOWN_VALUES[0])
    MYVECT.append(tmpvect)
    tmpvect = []
    tmpvect.append("bookmark2")
    tmpvect.append(LKNOWN_VALUES[1])
    MYVECT.append(tmpvect)
        
    def setUp (self):
        """ called before each test function from this class """
        import copy
        tmp = copy.deepcopy(self.MYVECT)
        
        self.pv = process_vectors (tmp, debug=0)

    
    def test_tfidf_known_values(self):
        """
        tf_idf should give known result with known input
        """

        vect = self.pv.tf_idf(self.pv.all_vector)
        
        # for each vector
        for i in range(len(vect)):
            for word, norm in vect[i][1]:
                self.assert_(self.HKNOWN_VALUES[i].has_key(word), \
                             'word "' + word + '" unknown')
                self.assert_(misc.float_equal(norm, self.HKNOWN_VALUES[i][word]), \
                             'Wrong norm for word "' + word + '"')
        

    def test_cv_known_values(self):
        """
        cut_vector should give known result with known input
        """
        vect = self.pv.cut_vector(self.pv.all_vector)

        # for each vector
        for i in range(len(vect)):
            for word in vect[i][1].keys():
                self.assert_(self.HKNOWN_VALUES_N_CV[i].has_key(word), \
                             'word "' + word + '" unknown')
                self.assert_(misc.float_equal(vect[i][1][word], 
                                        self.HKNOWN_VALUES_N_CV[i][word]), \
                             'Wrong norm for word "' + word + '"')


    def sanity_check(self):
        """
        global sanity check for do() function
        """
        
        vect = self.pv.do()

        # for each vector
        for i in range(len(vect)):
            for word in vect[i][1].keys():
                self.assert_(self.HKNOWN_VALUES_N[i].has_key(word), \
                             'word "' + word + '" unknown')
                self.assert_(misc.float_equal(vect[i][1][word],
                                              self.HKNOWN_VALUES_N[i][word]), \
                             'Wrong norm for word "' + word + '"')



#####
from sort_vectors_mc import *       
class SortVectorsKnownValues(unittest.TestCase):
    """
    This class check that sort_vectors_mc functions give known result
    with known input
    """

    HKNOWN_VALUES = [["b0",{}], ["b1",{}],
                    ["b2",{}], ["b3",{}]]
    HKNOWN_VALUES[0][1] = {
        'string':   3,
        'test':     2,
        'extract':  2,
        'occurs':   2,
        'text':     1,
        'function': 1,
        'ensure':   1,
        'word':     1,
        'words':    1,
        'count':    1,
        'right':    1,
        'number':   1,
        'narval':   1,
        'package':  1
        }
    HKNOWN_VALUES[1][1] = {
        'stupid':   3,
        'job':      3,
        'extract':  2,
        'occurs':   2,
        'text':     1,
        'function': 1,
        'world':    1,
        'magic':    1,
        'count':    2,
        'right':    1,
        'number':   4,
        'narval':   1,
        'package':  1,
        'beep':     1
        }
    HKNOWN_VALUES[2][1] = {
        'really':   3,
        'least':    2,
        'interest': 2,
        'occurs':   2,
        'text':     1,
        'function': 5,
        'ensure':   1,
        'word':     4,
        'words':    1,
        'count':    1,
        'right':    2,
        'number':   1,
        'narval':   1,
        'package':  1
        }
    HKNOWN_VALUES[3][1] = {
        'still':    3,
        'not':      2,
        'sure':     2,
        'then':     2,
        'try':      1,
        'it':       1,
        'ensure':   1,
        'word':     1,
        'words':    1,
        'right':    1,
        'number':   1,
        'narval':   1,
        'horn':     1
        }
    KNOWN_CENTERS = [{
        'occurs': 1.33333333333,
        'test': 0.666666666667,
        'interest': 0.666666666667,
        'ensure': 1.0,
        'try': 0.333333333333,
        'narval': 1.0,
        'package': 0.666666666667,
        'word': 2.0,
        'least': 0.666666666667,
        'count': 0.666666666667,
        'really': 1.0,
        'extract': 0.666666666667,
        'still': 1.0,
        'not': 0.666666666667,
        'words': 1.0,
        'it': 0.333333333333,
        'horn': 0.333333333333,
        'number': 1.0,
        'text': 0.666666666667,
        'sure': 0.666666666667,
        'string': 1.0,
        'right': 1.33333333333,
        'then': 0.666666666667,
        'function': 2.0
        },{
        'occurs': 2.0,
        'extract': 2.0,
        'world': 1.0,
        'count': 2.0,
        'stupid': 3.0,
        'job': 3.0,
        'beep': 1.0,
        'narval': 1.0,
        'number': 4.0,
        'text': 1.0,
        'package': 1.0,
        'magic': 1.0,
        'right': 1.0,
        'function': 1.0
        }]
    
    CENT_PTS = [deepcopy(HKNOWN_VALUES[0][1]),
                deepcopy(HKNOWN_VALUES[1][1])]
    ORDER = [[0, 1, 2], [3]]
    
    def setUp (self):
        """ called before each test from this class """
        self.svm = sort_vectors_mc(self.HKNOWN_VALUES,
                                   nb_class=2, debug=0)

        
    def test_sc_known_values(self):
        """
        _search_class should give known result with known input
        """
        vect = self.svm._search_class(self.CENT_PTS)

        for i in range (len(self.KNOWN_CENTERS)):
            for word in self.KNOWN_CENTERS[i].keys():
                self.assert_(misc.float_equal(vect[i][word],
                                         self.KNOWN_CENTERS[i][word]),
                             'Wrong point for center '+`i`)


    def sanity_check(self):
        """
        global sanity check for do() function
        """
        vect = self.svm.do()

        for i in range(len(vect)):
            j = 0
            for index in vect[i]:
                self.assert_(index == self.ORDER[i][j],
                        'Wrong classes')
                j = j+1



from myhtmllib import *
class MyhtmllibKnownValues(unittest.TestCase):
    """
    This class check that myhtmllib functions give
    known result with known input
    """

    DATA = """
<html>
  <head>
    <title>Sylvain Thenault</title>
  </head>
  <body>
    <h2>Infos</h2>
    <h1>Liens</h1>
    <h1>Travail en cours</h1>
    <ul>
      <li>Unit tests pour extensions infopal
      <li>Généralisation du tris des signets pour le journal</li>
    </ul>
  <b>bold</b>
  <i>italic</i>
  <em>emphasis</em>
  <h3>title3</h3>
  <h4>title4</h4>
  <h5>title5</h5>
  <h6>title6</h6>
  this should not be repeat
  </body>
</html>
"""
    KNOWN_VALUES =  """
Sylvain Thenault Sylvain Thenault Sylvain Thenault Infos Infos Infos
Liens Liens Liens Travail en cours Travail en cours Travail en cours
 
Unit tests pour extensions infopal
Généralisation du tris des signets pour le journal
 
bold bold bold italic italic italic emphasis emphasis emphasis title3
title3 title3 title4 title4 title4 title5 title5 title5 title6 title6
title6 this should not be repeat 
"""

    def sanity_check(self):
        from string import split
        from formatter import AbstractFormatter, DumbWriter
        data = StringIO()
        form = AbstractFormatter(DumbWriter(data))
        p = MyHTMLParser(form)
        p.feed(self.DATA)
        p.close()
        data.seek(0)
        i = 0
        tab  = split(self.KNOWN_VALUES)

        for word in split(data.read()):
            self.assert_(word == tab[i], 'Bad output for word "'+word+'"') 
            i = i + 1
        data.close()

        

from guess_language import *
class GuessLanguageKnownValues(unittest.TestCase):
    """
    This class check that guess_language functions give
    known result with known input
    """
 
    ENGLISH = """
    Hello, hi
    This is a piece of english text. Guess_language should find
    the right language even if there is a french (or antoher language)
    cite.
    Will it be abble to do this, voila une question intéressante!
    Sure he will
    Take care and have fun, be carefull with the big bad wolf (!?)
    I would notice it's much more difficult to write an enough long
    text by myself than only take a web page in the correct langugage
    (see below) ...
    """
    FRENCH = """
    Bonjour, salut
    Voila un morceau de texte en francais. Guess_language devrait
    trouver cette langue même avec quelques citations dans une langue
    étrangère.
    En sera Thenault'il capable, that's the question...
    Je note egalement que même en francais je ne suis pas foutu de
    trouver quelque chose d'intéressant a marquer, j'aurais mieux fait
    de prendre une page web.
    """
    ITALIAN = """
    Castello (Il) - Storia, attualità e cultura della città e della
    zona.  il Giornale di Cava - Mensile di politica, attualità e
    cultura della Città di Cava de' Tirreni.  Mattino (Il) - Contiene
    gli articoli dell'edizione nazionale e di quelle locali. Dispone
    di un motore di ricerca interno.  Opinione di Stabia (L') -
    Quotidiano indipendente, con sezioni di attualità, storia e sport.
    """ 
    GERMAN = """
    Bauer Mergers & Acquisitions AG - berät beim Kauf und
    Verkauf von Unternehmen und Beteiligungen.  BioAgency AG -
    fördert, unterstützt und betreut die Gründung neuer sowie die
    Ansiedlung bereits existierender Biotechnologiefirmen und anderer
    Life-Science-Unternehmen in Hamburg und Norddeutschland.  Capital
    Invest - vermittlet Investoren für expandierende
    zukunftsorientierte Unternehmen.  Carlyle Group Deutschland, The -
    ist als Eigenkapitalinvestor auf die Generierung, Strukturierung
    und Finanzierung von Management Buy-outs, Mehrheitsbeteiligungen
    und strategischen Minderheitsbeteiligungen sowie von
    Konsolidierungstransaktionen und Wachstumsfinanzierungen
    spezialisiert
    """
    SPANISH = """
    Atiza - Bares, conciertos, discos, grupos y noticias.  Big-Bang -
    Portal de ayuda para músicos.  Dance Loud Company - Guía de
    discotecas de música dance con correo electrónico de DJ y tiendas.
    Discografías - Discografía completa de varios artistas y enlaces a
    sus sitios.  Doctor Music - Agenda de conciertos, noticias,
    entrevistas, charlas, mediateca y tablón de anuncios.  Es Música -
    Novedades, actualidad musical, conciertos, artistas y archivos
    musicales.  Fregadero Sintético - Acceso a conciertos, música
    alternativa, discos, maquetas y al Festival Asituna Rock.  Grupos
    y Conciertos - Grupos y solistas, noticias, foros, sorteos, salas
    de concierto, discográficas y música internacional.  GuíaMusical -
    La escena musical en España, organizada por comunidades.  Madrid
    Music - Música de temas generales con especial atención a lo que
    sucede en Madrid.  Mai Records - Guía de archivos de música
    alternativa y no comercial en distintos formatos con tablón de
    anuncios, comentarios y enlaces.  Mucho Ruido - Top 10, noticias,
    estrenos, agenda, descargas y más.  Musices.Com - Guía de la
    música profesional en España clasificado por artistas, productores
    y agencias de espectáculos y publicidad.  Popwire - Buscador de
    grupos musicales, referencias a los grupos de moda y a los nuevos
    talentos.  Portal Latino - Centro de recursos creado por la
    Sociedad Digital de Autores y Editores para promocionar la música
    y otras expresiones culturales latinas desde las perspectiva de
    sus
    """
    PORTUGUESE = """
    Attac Brasil - Associação por uma Taxação das Transações
    financeiras para Apoio aos Cidadãos.  Banco Federativo - BNDES -
    Biblioteca de estudos e estatísticas sobre a economia do setor
    público, para ajudar a entender e a aplicar corretamente a Lei de
    Responsabilidade Fiscal.  Chega de Confisco - Campanha dos
    auditores fiscais pela correção dos valores a deduzir na tabela do
    Imposto de Renda.  Instituto Brasileiro de Planejamento Tributário
    - Apresenta seus estudos, notícias e editorial sobre impostos e
    reforma tributária.  IPTU Certo - Orienta quanto ao cálculo
    correto do imposto.  Refis Fácil - Nacir Sales apresenta o
    Movimento Nacional de Adequação do Refis e traz orientações e a
    relação de normas.  Result Online - Banco de dados de legislação
    tributária.  Secretaria da Receita Federal - Impostos, estudos
    tributários, declaração e restituição do IRPF e IRPJ, registro de
    CPF, CNPJ etc.  Sintegra - Sistema integrado de informações sobre
    operações interestaduais com mercadorias e as implicações no ICMS.
    """
    
    def sanity_check(self):
        self.assert_(
            identify_language_from_buffer(self.ENGLISH) == 'en',
            'ENGLISH')
        self.assert_(
            identify_language_from_buffer(self.FRENCH) == 'fr',
            'FRENCH')
        self.assert_(
            identify_language_from_buffer(self.ITALIAN) == 'it',
            'ITALIAN')
        self.assert_(
            identify_language_from_buffer(self.GERMAN) == 'de',
            'GERMAN')
        self.assert_(
            identify_language_from_buffer(self.SPANISH) == 'es',
            'SPANISH')
        self.assert_(
            identify_language_from_buffer(self.PORTUGUESE) == 'pt',
            'PORTUGUESE')



        
if __name__=='__main__':
    
    testsuite = unittest.TestSuite()
    
    testsuite.addTest(SourceVectorKnownValues("test_cwo_known_values"))
    testsuite.addTest(SourceVectorKnownValues("test_scv_known_values"))
    testsuite.addTest(SourceVectorKnownValues("test_cv_known_values"))
    testsuite.addTest(SourceVectorKnownValues("sanity_check"))
    
    testsuite.addTest(ProcessVectorsKnownValues("test_tfidf_known_values"))
    testsuite.addTest(ProcessVectorsKnownValues("test_cv_known_values"))
    testsuite.addTest(ProcessVectorsKnownValues("sanity_check"))
    
    testsuite.addTest(SortVectorsKnownValues("test_sc_known_values"))
    testsuite.addTest(SortVectorsKnownValues("sanity_check"))
    
    testsuite.addTest(MyhtmllibKnownValues("sanity_check"))
    
    testsuite.addTest(GuessLanguageKnownValues("sanity_check"))

    test = unittest.TextTestRunner()
    # uncomment next line to write tests results in a file
    #test.__init__(open('unittest.log','w+'))    

    test.run(testsuite)
