Changeset 162:ca1529c62df9 for classifier/classifier.py

Timestamp:

08/21/07 19:12:16 (11 months ago)

Author:

Tarek Ziad?? <tarek@…>

Message:

fixes

Files:

• classifier/classifier.py (modified) (6 diffs)

classifier/classifier.py

@@ -31 +31 @@
         self.backend = backend
         self.tokenizer = tokenizer
+        self._learnt = True
+        self._probs = None
+
         if options is None:
             self.options = {'lang': self.language}
@@ -42 +45 @@
         wich means: store words in categories
         """
+        self._learnt = True
         self.backend.add_category(name=category)
         data = self.tokenizer.transform(data, self.options)
@@ -52 +56 @@
         wich means: remove words from categories
         """
+        self._learnt = True
         data = self.tokenizer.transform(data, self.options)
         for element in data:
-            self.backend.delWord(element, category)
+            self.backend.del_word(element, category)
 
     def guess(self, data):
@@ -62 +67 @@
 
         # XXX this will be cached
-        probabilities = self._buildWordProbabilities()
+        if self._learnt:
+            self._probs = self._buildWordProbabilities()
+        else:
+            if self._probs is None:
+                self._probs = self._buildWordProbabilities()
+
+        probabilities = self._probs
+        self._learnt = False
 
         res = {}
@@ -124 +136 @@
     def _buildWordProbabilities(self, language=None):
         probs = {}
+        corpus_size = self.corpusSize(language)
+        words = list(self.backend.list_words(language, complete=True))
         for cat in self.backend.list_categories():
-            probs[cat] = self._buildCategoryWordProbabilities(cat, language)
+
+            probs[cat] = self._buildCategoryWordProbabilities(cat, language,
+                                                              corpus_size, words)
         return probs
 
-    def _buildCategoryWordProbabilities(self, category, language=None):
+    def _buildCategoryWordProbabilities(self, category, language=None,
+                                        corpus_size=None, words=None):
         """Merges corpora and computes probabilities
 
         XXX to be cached later (invalidation on word adding)
         """
+        if corpus_size is None:
+            corpus_size = self.corpusSize(language)
+        if words is None:
+            words = self.backend.list_words(language, complete=True)
         if language is None:
             language = self.language
-        corpus_size = self.corpusSize(language)
+
         category_size = float(self.categorySize(category, language))
         them_count = float(max(corpus_size - category_size, 1))
         probabilities = {}
-        words = self.backend.list_words(language, complete=True)
 
         for word in words:
-            if category not in word[1][1].keys():
+            the_word = word[1][1]
+
+            if category not in the_word.keys():
                 continue
 
@@ -149 +171 @@
                 continue
 
-            cat_word_count = float(word[1][1][category])
-            other_count = cat_word_count - word_count
+            cat_word_count = float(the_word[category])
+            other_count = word_count - cat_word_count
 
             if category_size == 0: