X-Git-Url: https://git.kaliko.me/?a=blobdiff_plain;f=sima%2Flib%2Fsimastr.py;h=56edbb4d7e68b82a816ad2d30cb6a99698d78864;hb=08d0003d11906338f32a54891edd13fe8316abcd;hp=7e7668c891979562b3232c614a053b345866f42e;hpb=c1bda032095902bdcd183c530a9c4de28f3c828a;p=mpd-sima.git

diff --git a/sima/lib/simastr.py b/sima/lib/simastr.py
index 7e7668c..56edbb4 100644
--- a/sima/lib/simastr.py
+++ b/sima/lib/simastr.py
@@ -1,5 +1,4 @@
 # -*- coding: utf-8 -*-
-
 #
 # Copyright (c) 2009, 2010, 2013 Jack Kaliko <kaliko@azylum.org>
 #
@@ -18,18 +17,17 @@
 #  If not, see <http://www.gnu.org/licenses/>.
 #
 
-"""
+r"""
 SimaStr
 
 Special unicode() subclass to perform fuzzy match on specific strings with
 known noise.
 
-Artist names often contain a leading 'The ' which might, or might not be
-present. Some other noise sources in artist name are 'and' words :
-    'and'/'&'/'n'/'N'.
-
-The SimaStr() object removes these words and compute equality on "stripped"
-strings.
+ * SimaStr() object removes specific patterns from the string
+ * Diacritic are removed
+ * Equality test is done on lower-cased string
+ * Equality test is not an exact comparison, the levenshtein edition distance
+   between stripped and filtered strings is used
 
 >>> from simastr import SimaStr
 >>> art0 = SimaStr('The Desert Sessions & PJ Harvey')
@@ -38,9 +36,14 @@ strings.
 >>> True
 >>> art0 == 'Desert Sessions And PJ Harvey'
 >>> True
+>>> # diacritic filter + levenshtein  example
+>>> art0 = sima.lib.simastr.SimaStr('Hubert Félix Thiéphaine')
+>>> art1 = sima.lib.simastr.SimaStr('Hubert-Felix Thiephaine')
+>>> art0 == art1
+>>> True
 >>>
 
-Current stripped word patterns (usually English followed by French andx
+Current stripped word patterns (usually English followed by French and
 Spanish alternatives)
     leading (case-insensitive):
             "the","le","la","les","el","los"
@@ -50,10 +53,9 @@ Spanish alternatives)
             combination of "[- !?\.]+" "\(? ?[Ll]ive ?\)?"
 
 
-Possibility to access to stripped string :
+Possibility to access to stripped string:
 
 >>> art0 = SimaStr('The Desert Sessions & PJ Harvey')
->>> art.stripped
 >>> print (art0, art0.stripped)
 >>> ('The Desert Sessions & PJ Harvey', 'Desert Sessions PJ Harvey')
 
@@ -63,10 +65,13 @@ TODO:
 """
 
 __author__ = 'Jack Kaliko'
-__version__ = '0.3'
+__version__ = '0.4'
 
 # IMPORTS
-from re import (compile, U, I)
+import unicodedata
+from re import compile as re_compile, U, I
+
+from ..utils.leven import levenshtein_ratio
 
 
 class SimaStr(str):
@@ -74,6 +79,8 @@ class SimaStr(str):
     Specific string object for artist names and song titles.
     Here follows some class variables for regex to run on strings.
     """
+    diafilter = True
+    leven_ratio = 0.82
     regexp_dict = dict()
 
     # Leading patterns: The Le Les
@@ -86,20 +93,24 @@ class SimaStr(str):
     # Trailing patterns: ! ? live
     # TODO: add "concert" key word
     #       add "Live at <somewhere>"
-    regexp_dict.update({'trail': '([- !?\.]|\(? ?[Ll]ive ?\)?)'})
+    regexp_dict.update({'trail': r'([- !?\.]|\(? ?[Ll]ive ?\)?)'})
 
-    reg_lead = compile('^(?P<lead>%(lead)s )(?P<root0>.*)$' % regexp_dict, I | U)
-    reg_midl = compile('^(?P<root0>.*)(?P<mid> %(mid)s )(?P<root1>.*)' % regexp_dict, U)
-    reg_trail = compile('^(?P<root0>.*?)(?P<trail>%(trail)s+$)' % regexp_dict, U)
+    reg_lead = re_compile('^(?P<lead>%(lead)s )(?P<root0>.*)$' % regexp_dict, I | U)
+    reg_midl = re_compile('^(?P<root0>.*)(?P<mid> %(mid)s )(?P<root1>.*)' % regexp_dict, U)
+    reg_trail = re_compile('^(?P<root0>.*?)(?P<trail>%(trail)s+$)' % regexp_dict, U)
 
     def __init__(self, fuzzstr):
         """
         """
-        str().__init__(fuzzstr)
         self.orig = str(fuzzstr)
         self.stripped = str(fuzzstr.strip())
         # fuzzy computation
         self._get_root()
+        if self.__class__.diafilter:
+            self.remove_diacritics()
+
+    def __new__(cls, fuzzstr):
+        return super(SimaStr, cls).__new__(cls, fuzzstr)
 
     def _get_root(self):
         """
@@ -121,13 +132,23 @@ class SimaStr(str):
             #print sea.groupdict()
             self.stripped = sea.group('root0')
 
+    def remove_diacritics(self):
+        """converting diacritics"""
+        self.stripped = ''.join(x for x in
+                                unicodedata.normalize('NFKD', self.stripped)
+                                if unicodedata.category(x) != 'Mn')
+
     def __hash__(self):
         return hash(self.stripped)
 
     def __eq__(self, other):
         if not isinstance(other, SimaStr):
-            return hash(self) == hash(SimaStr(other))
-        return hash(self) == hash(other)
+            other = SimaStr(other)
+        levenr = levenshtein_ratio(self.stripped.lower(),
+                                   other.stripped.lower())
+        if hash(self) == hash(other):
+            return True
+        return levenr >= self.__class__.leven_ratio
 
     def __ne__(self, other):
         if not isinstance(other, SimaStr):