"""This file sort many url from :
    - arguments
    - and/or a file (-i file) (in the file, it must have one url per line).
"""

# Copyright (c) 2001 LOGILAB S.A. (Paris, FRANCE).
# http://www.logilab.fr/ -- mailto:contact@logilab.fr
#
# This program is free software; you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free Software
# Foundation; either version 2 of the License, or (at your option) any later
# version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along with
# this program; if not, write to the Free Software Foundation, Inc.,
# 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.


# FIXME : pychecker

from narval.lib import load_extension_module
misc  = load_extension_module('classify', 'misc')

class process_vectors:
    """This class .
       The source 'data' must be a StringIO.
       The vector should be :
        - a full vector, all words find in source and their occurrence:
          (only_dico=1, list_vector).
        - a short vector, representative vector of source, 'first' words
          find in source and their normalizing occurrence (list_repr_vect).
       A debug option is available.
    """
    
    def __init__(self, all_vector, debug=0):
        self.debug = debug
        self.N = 2000
        # transform [[<>,{'':val}]...] to [[<>,['',val]]...]
        for i in range(len(all_vector)):
            tmp_list= []
            for key in all_vector[i][1].keys():
                # foreach duo in vector of current document
                duo = [key, all_vector[i][1][key]]
                tmp_list.append(duo)
            all_vector[i][1] = tmp_list
        self.all_vector = all_vector


    def do(self):
        self.all_vector = self.tf_idf(self.all_vector)
        self.all_repr_vect = self.cut_vector(self.all_vector)
        return self.all_repr_vect


    def tf_idf(self, all_vector):
        """        
        tf-idf algorithm
        """
        # create commun word dictionnary
        com_words = {}
        # Search common words in all bookmarks
        for doc in all_vector:
            for word, occ in doc[1]:
                if com_words.has_key(word):
                    com_words[word] = com_words[word] + 1
                else:
                    com_words[word] = 1
        
        # map tf-idf algorithm ti/T * log(N/ni)
        import math
        N = float(len(all_vector))
        
        for i in range(N):
            # foreach document
            T = float(len(all_vector[i][1]))

            for j in range(len(all_vector[i][1])):
                 # foreach duo in vector of current document
                 duo = [all_vector[i][1][j][0]]
                 duo.append(all_vector[i][1][j][1]/T * \
                            math.log(N/com_words[all_vector[i][1][j][0]]))
                 all_vector[i][1][j] = duo
                 
            # Sort foreach document, its vector
            all_vector[i][1].sort(misc.cmp_eq_words)
        
        return all_vector  # [[url, []]]


    def cut_vector(self, all_vector):
        import math
        nb_word = 30
        all_repr_vect = []
        # foreach document
        for doc in all_vector:
            
            if len(doc[1]) >= nb_word:
                l = nb_word
            else:
                l = len(doc[1])

            # build vector and
            # perform vector Euclidienne norm
            tmpvect = []
            for i in range(l):
                tmpvect.append(doc[1][i][1])
            vect_norm = misc.norm(tmpvect)    

            # Create the representative normalized vector
            vect = {}
            if vect_norm != 0:
                if len(doc[1]) >= nb_word:
                    for word, occ in doc[1][:nb_word]:
                        vect[word] = (occ*self.N)/vect_norm
                else:
                    for word, occ in doc[1]:
                        vect[word] = (occ*self.N)/vect_norm
                        
            all_repr_vect.append([doc[0], vect])
            
            if self.debug:
                print '\n'
                print doc[0]
                print l
                print vect_norm
                print vect

        return all_repr_vect # [[url, {}]]

