Changeset 102:917b0f7d3576 for atomisator/filters

Timestamp:

05/30/07 16:47:14 (18 months ago)

Author:

Tarek Ziad?? <tarek@…>

Message:

fixes

Location:

atomisator/filters

Files:

• BayesCore/classifier.py (modified) (2 diffs)
• BayesCore/data/bayes.db (modified) (previous)
• BayesCore/storage.py (modified) (1 diff)
• BayesCore/tokenizer/filters.py (modified) (11 diffs)
• BayesCore/tokenizer/normalized.fr.txt (modified) (1 diff)
• BayesCore/tokenizer/normalized.txt (modified) (1 diff)
• bayes.py (modified) (4 diffs)

atomisator/filters/BayesCore/classifier.py

@@ -1 @@
-# -*- coding: iso-8859-15 -*-
+# -*- coding: utf8 -*-
 # Copyright (c) 2006 Nuxeo SAS <http://nuxeo.com>
-# Authors : Tarek Ziadé <tziade@nuxeo.com>
+# Authors : Tarek Ziadé <tziade@nuxeo.com>
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 2 as published
@@ -63 +63 @@
         # XXX this will be cached
         probabilities = self._buildWordProbabilities()
+
         res = {}
         for category_name in probabilities:

atomisator/filters/BayesCore/storage.py

@@ -25 +25 @@
     def __init__(self, sqluri):
         self._words, self._catwords = self._get_mapper(sqluri)
+        self._sqluri = sqluri
+
         try:
             self._words.create()

atomisator/filters/BayesCore/tokenizer/filters.py

@@ -1 @@
-# -*- coding: iso-8859-15 -*-
+# -*- coding: utf8 -*-
 # Copyright (c) 2006 Nuxeo SAS <http://nuxeo.com>
-# Authors : Tarek Ziadé <tziade@nuxeo.com>
+# Authors : Tarek Ziadé <tziade@nuxeo.com>
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 2 as published
@@ -43 +43 @@
 
     def transform(self, text, options):
-        tokenizers = ('splitter', 'stopwords', 'normalizer', 'stemmer')
+        tokenizers = ('normalizer', 'splitter', 'stopwords', 'stemmer')
         return applyFilters(tokenizers, text, options)
 
@@ -61 +61 @@
             self.charset = options['charset']
         else:
-            self.charset = 'ISO-8859-15'
+            self.charset = 'utf8'
 
     def getFinalState(self, result):
@@ -76 +76 @@
     name = 'splitter'
 
-    char_list = ',;:/\'"#?!.-=+_`|()[]{}<>~&§%'
+    char_list = ',;:/\'"#?!.-=+_`|()[]{}<>~&§%'
 
     def getName(self):
@@ -108 +108 @@
                                            Splitter().split(text)])
 
-        text = ''.join([self._cleanChar(char) for char in text])
+        #text = ''.join([self._cleanChar(char) for char in text])
+        text = ' '.join(text)
+
+        for char in self.char_list:
+            text = text.replace(char, ' ')
 
         result = text.split()
@@ -124 +128 @@
 
     name = 'stopwords'
+    treshold = 2
 
     def getName(self):
@@ -152 +157 @@
         lang = options['lang']
         stopwords = self._getStopWords(lang)
-        return [word for word in text if word not in stopwords]
+        return [word for word in text if (word not in stopwords
+                and len(word) > self.treshold)]
 
 registerFilter(StopWords())
@@ -190 +196 @@
             else:
                 return car
-        normalized = [normalized(car) for car in word]
-        return ''.join(normalized)
+
+        #normalized = [normalized(car) for car in word]
+        for car in normalizer:
+            word = word.replace(car, normalizer[car])
+
+        return word
+        #''.join(normalized)
 
     def transform(self, text, options):
@@ -222 +233 @@
 
     name = 'stemmer'
-    charset = 'ISO-8859-15'
+    charset = 'utf8'
 
     def getName(self):
@@ -262 +273 @@
         def checktype(element):
             if isinstance(element, str):
-                return element.decode(charset)
+                return element.decode(charset, 'replace')
             return element
 
@@ -281 +292 @@
 
 registerFilter(Stemmer())
+

atomisator/filters/BayesCore/tokenizer/normalized.fr.txt

@@ -2 +2 @@
 # will probably get TextIndexNG normalizer
 # collection next
-# and be fine-tuned later
-é e
-è e
-à a
-ç c
-ù u
+#Â and be fine-tuned later
+é e
+Ú e
+Ã  a
+ç c
+ù u
 # to be completed

atomisator/filters/BayesCore/tokenizer/normalized.txt

@@ -2 +2 @@
 # will probably get TextIndexNG normalizer
 # collection next
-# and be fine-tuned later
-é e
-è e
-à a
-ç c
-ù u
+#Â and be fine-tuned later
+é e
+Ú e
+Ã  a
+ç c
+ù u
 # to be completed

atomisator/filters/bayes.py

@@ -49 +49 @@
 SQLURI = 'sqlite:///filters/BayesCore/data/bayes.db'
 
-def bayesian(entry, entries):
+def bayesian(entry, entries, sqluri=None):
     """uses bayesian inference over entries"""
     content = entry['content'].encode('utf8')
@@ -59 +59 @@
 
     data = '%s %s' % (content, title)
-    classifier = BayesClassifier(LANG, SQLStorage(SQLURI), AllFilters())
+
+    if sqluri is None:
+        sqluri = SQLURI
+
+    classifier = BayesClassifier(LANG, SQLStorage(sqluri), AllFilters())
 
     # let's test the entry
@@ -67 +71 @@
         return True
 
+
     return  result[0][0] == 'nojunk'
 
 register_filter(bayesian)
 
-def bayesian_learn(entry):
+def bayesian_learn(entry, sqluri=None, answer=None):
     """uses bayesian inference over entries"""
     content = entry['content'].encode('utf8')
@@ -80 +85 @@
         title = ''
 
-    print 'title : %s' % title
+
+    if sqluri is None:
+        sqluri = SQLURI
 
     data = '%s %s' % (content, title)
-    classifier = BayesClassifier(LANG, SQLStorage(SQLURI), AllFilters())
 
-    res = raw_input("Interesting (type 'm' for more) ? (y/n)  ")
-    res = res.strip().lower()
+    classifier = BayesClassifier(LANG, SQLStorage(sqluri), AllFilters())
 
-    if res == 'm':
-        print content
-        res = raw_input("Interesting ? (y/n)  ")
+    if answer is None:
+        print 'title : %s' % title
 
+        res = raw_input("Interesting (type 'm' for more) ? (y/n)  ")
+        answer = res.strip().lower()
 
-    print 'Learning...'
-    if res.strip().lower() in ('y', 'yes'):
+        if answer == 'm':
+            print content
+            answer = raw_input("Interesting ? (y/n)  ")
+
+        print 'Learning...'
+
+    if answer.strip().lower() in ('y', 'yes'):
         classifier.learn(data, 'nojunk')
     else: