"""This file defines one class : tranf_source_vector
   and a test function to this class.
"""

# Copyright (c) 2001 LOGILAB S.A. (Paris, FRANCE).
# http://www.logilab.fr/ -- mailto:contact@logilab.fr
#
# This program is free software; you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free Software
# Foundation; either version 2 of the License, or (at your option) any later
# version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along with
# this program; if not, write to the Free Software Foundation, Inc.,
# 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.

from narval.lib import load_extension_module
misc  = load_extension_module('classify', 'misc')

class tranf_source_vector:
    """This class transforms a source to vector (words: occurrence).
       The source 'data' must be a StringIO.
       The vector should be :
        - a full vector, all words find in source and their occurrence:
          (only_dico=1, list_vector).
        - a short vector, representative vector of source, 'first' words
          find in source and their normalizing occurrence (list_repr_vect).
       A debug option is available.
    """
    
    def __init__(self, use_engl_dico=1, nb_mini_letters=4):
         self.nb_word = 0       # word total number in the text
         self.nb_diff_word = 0  # different word number in the text
         self.nb_repr_word = 0  # word number in the representative list
         self.nb_max_occurs = 0 # occurs number for the word
                                # that's most in the text
         self.indice_num_l = -1 # indice of the list_vector that represents
                                # the left limit of the text theme
         self.indice_num_r = -1 # ... right ...
         self.list_repr_vect={} # vector representing text
         self.N = 2000          # length of vector
         self.nb_mini_letters = nb_mini_letters
                                # define the number letters minimums for words
         self.use_engl_dico = use_engl_dico
                                # default use english dico
         self.engl_dico = {}    # dico of word do not get
         #self.vect_norm        # vector norm (in debug mode)

         if self.use_engl_dico:
             # Init engl dictionary
             import sys
             from narval import config
             from os.path import normpath, join
             path = normpath(join(
                 str(config.get_home()),
                 'extensions/language/',
                 'engl_dico.txt'))
             try:
                 f = open(path ,'r')
             except IOError, msg:
                 print msg
                 raise ReadError
             engl_data = f.readlines()
             if f is not sys.stdin:
                 f.close()
             
             for i in range(len(engl_data)):
                 if len(engl_data[i]) > 1:
                     if engl_data[i][-1] == '\n':
                         self.engl_dico[engl_data[i][:-1]] = 0
                     else:                               
                         self.engl_dico[engl_data[i]] = 0



    def do(self, data, only_dico=0, debug=0):
        """
        Do run this class
        if only_dico => return full vector (list_vector)
        if not only_dico => return representative vector of source
                                                  (list_repr_vect)
        /!\ return a {}  /!\
        The source 'data' must be a StringIO.
        """
        # Test parameter
        try:
            data.seek(0)
        except:
            print "In tranf_source_vector class : 'data' must be a StringIO"
            return None
            #import sys
            #sys.exit(1)
        
        list_vector = self.count_words_occurs(data)
        data.close()
        
        if not only_dico:
            self.search_cut_vector(list_vector)
            self.list_repr_vect = self.cut_vector(list_vector, debug)
            if debug :
                # Print the full vector (list_vector)
                print 'Indice', "('word': nb_occ)", \
                      'normalize_occurrence / ', self.N
                for i in range(self.nb_diff_word) :
                    print i, list_vector[i], \
                          int(float(list_vector[i][1])*200/self.vect_norm)

                print '*-' *10
                print 'Cut dictionary :',self.indice_num_l, self.indice_num_r,\
                      ' => ', self.nb_repr_word+1, 'word(s)'
                print 'source vector => ', self.list_repr_vect                
            return self.list_repr_vect
        else:
            if debug :
                # Print the full vector (list_vector)
                print 'Indice', "('word': nb_occ)"
                for i in range(self.nb_diff_word) :
                    print i, list_vector[i]
            # transform [['',val]...] to [{'':val}...]
            tmp_list= {}
            for word, occurs in list_vector:
                tmp_list[word] = occurs
            return tmp_list



    def count_words_occurs(self, data):
        """
        Count occurs number of each word.
        'data' must be a StringIO
        return decreasing vector
        This fct use self: 'nb_word', 'nb_diff_word', 'nb_max_occurs', 
        nb_mini_letters'
        """
        import re
        import string
         
        data.seek(0)
        
        # select words
        reg_exp = re.compile(
            r'[\007-\015\040-\057\072-\077\134-\140\173-\176]*')
        word_list = re.split(reg_exp, string.lower(data.read()))
        
        # count the occurs word number
        unit = 1
        dict_word = {}
        for word in word_list :
            if len(word) > 0:
                self.nb_word = self.nb_word + 1
                #self.nb_word = self.nb_word + len(word) + 1
            if len(word) >= self.nb_mini_letters and not self.engl_dico.has_key(word):
                try:
                    dict_word[word] = dict_word[word] + unit
                except:
                    # first time
                    dict_word[word] = unit
         
        # dict => list and sort
        self.nb_diff_word = len(dict_word)
        list_vect = dict_word.items()
        list_vect.sort(misc.cmp_eq_words)
        try:
            self.nb_max_occurs = list_vect[0][1]
        except:
            self.nb_max_occurs = 0

        return list_vect


    def search_cut_vector(self, list_vect):
        """
        Search the indice number in list vector to keep main words.
        return indice_number of list of vector         
        This fct use self : 'nb_diff_word', 'nb_max_occurs', 'indice_num_l _r'
        """
        if self.nb_max_occurs == 0:
            self.indice_num_l = 0
            self.indice_num_r = 0
        else:
            # Set indice_num_r
            self.indice_num_r = 25
            # Check indice_num_r
            if self.nb_diff_word < self.indice_num_r :
                self.indice_num_r = self.nb_diff_word

            # Set indice_num_l
            if self.nb_diff_word > 100:
                if self.nb_diff_word > 1000:
                    self.indice_num_l = 2
                else: self.indice_num_l = 1
            else: self.indice_num_l = 0
            # Check indice_num_l
            if self.nb_diff_word < self.indice_num_l or self.indice_num_l < 0:
                self.indice_num_l = 0


    def cut_vector(self, list_vect, debug=0):
        """
        Cut vector and compute its coordinates
        return the representative vector list
        This fct use self: 'nb_diff_word', 'nb_repr_word', 'N',
        'indice_num_l _r', 'vect_norm', 'vect_norm'
        """

        self.nb_repr_word = self.indice_num_r - self.indice_num_l
        list_repr_vect = {}

        # build vector with words occurs and
        # perform vector Euclidienne norm
        tmpvect = []
        for i in range(self.indice_num_l, self.indice_num_r):
            tmpvect.append(list_vect[i][1])
        vect_norm = misc.norm(tmpvect)
        
        # Create the representative normalized vector
        for i in range(self.indice_num_l, self.indice_num_r):
            list_repr_vect[list_vect[i][0]] = \
                                            int((float(list_vect[i][1])*self.N)/vect_norm)
        if debug :
            self.vect_norm = vect_norm

        return list_repr_vect


    def save(self, list_vect, write_file):
         """
         Save the vector to coordonate in file
         """
         import sys
         try:
              f = open(write_file, 'w')
         except IOError, msg:
              print write_file, ":", msg
              sys.exit(1)

         # data format in file
         for i in range(len(list_vect)) :
              f.write(str(i))
              f.write('       ')
              f.write(str(list_vect[i][1]))
              f.write('\n')
         f.close()


class ReadError(Exception):
    """This class defines a read error when it's impossible to read an file."""
    pass




# A little test suite
def test(args = None):
     import sys, getopt, myhtmllib
     
     if not args:
         args = sys.argv[1:]
        
     optlist, args = getopt.getopt(args, 'o:')
     output = None
     for attr, value in optlist:
          if attr == '-o':
               output = value
     
     if len(args) != 1:
          print 'COMMAND: [-o file.in] url'
          print '  file.in for graphics representation'
          print "  (ex: to visualize, use: gnuplot; plot [0:100] 'file.in')"
          sys.exit(1)     
     
     try:
         data = myhtmllib.parse_html_url(args[0], debug=0)
     except myhtmllib.ReadError:
         sys.exit(1)
     
     tsv = tranf_source_vector()
     
     if output:
         try:
             list_vector = tsv.do(data, debug=1, only_dico=1)
         except myhtmllib.ReadError:
             sys.exit(1)
         tsv.save(list_vector, output)
     else:
         try:
             tsv.do(data, debug=1, only_dico=0)
         except myhtmllib.ReadError:
             sys.exit(1)
    

        
if __name__ == '__main__':
    test() 
