| 1 | # -*- coding: iso-8859-15 -*- |
|---|
| 2 | # Copyright (c) 2006 Nuxeo SAS <http://nuxeo.com> |
|---|
| 3 | # Authors : Tarek Ziadé <tziade@nuxeo.com> |
|---|
| 4 | # This program is free software; you can redistribute it and/or modify |
|---|
| 5 | # it under the terms of the GNU General Public License version 2 as published |
|---|
| 6 | # by the Free Software Foundation. |
|---|
| 7 | # |
|---|
| 8 | # This program is distributed in the hope that it will be useful, |
|---|
| 9 | # but WITHOUT ANY WARRANTY; without even the implied warranty of |
|---|
| 10 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
|---|
| 11 | # GNU General Public License for more details. |
|---|
| 12 | # |
|---|
| 13 | # You should have received a copy of the GNU General Public License |
|---|
| 14 | # along with this program; if not, write to the Free Software |
|---|
| 15 | # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA |
|---|
| 16 | # 02111-1307, USA. |
|---|
| 17 | # |
|---|
| 18 | # $Id: __init__.py 45713 2006-05-18 13:57:32Z ogrisel $ |
|---|
| 19 | |
|---|
| 20 | from filters import applyFilters |
|---|
| 21 | from filters import AllFilters |
|---|
| 22 | |
|---|
| 23 | def tokenize(data, options=None): |
|---|
| 24 | """ default tokenizer """ |
|---|
| 25 | filters = ('splitter', 'stopwords', 'normalizer', 'stemmer') |
|---|
| 26 | return applyFilters(filters, data, options) |
|---|
