Improved ETag support, add some stats

[mpd-sima.git] / sima / lib / simaecho.py

diff --git a/sima/lib/simaecho.py b/sima/lib/simaecho.py
index 8c83adaba76165cb202b99db0bb753866c6eb99a..2706513b47c5f8d0ad2114ed316769fa493b2774 100644 (file)
--- a/sima/lib/simaecho.py
+++ b/sima/lib/simaecho.py
@@ -21,51 +21,58 @@
 Consume EchoNest web service
 """
 
 Consume EchoNest web service
 """
 

-__version__ = '0.0.1'
+__version__ = '0.0.2'

 __author__ = 'Jack Kaliko'
 
 
 __author__ = 'Jack Kaliko'
 
 

-from datetime import datetime, timedelta
+from datetime import timedelta

 
 

-from requests import get, Request, Timeout, ConnectionError
+from requests import Session, Request, Timeout, ConnectionError

 
 from sima import ECH
 from sima.lib.meta import Artist
 from sima.lib.track import Track
 
 from sima import ECH
 from sima.lib.meta import Artist
 from sima.lib.track import Track

+from sima.lib.http import CacheController

 from sima.utils.utils import WSError, WSNotFound, WSTimeout, WSHTTPError
 from sima.utils.utils import WSError, WSNotFound, WSTimeout, WSHTTPError

-from sima.utils.utils import getws, Throttle, Cache, purge_cache
+from sima.utils.utils import getws, Throttle

 if len(ECH.get('apikey')) == 23:  # simple hack allowing imp.reload
     getws(ECH)
 
 # Some definitions
 if len(ECH.get('apikey')) == 23:  # simple hack allowing imp.reload
     getws(ECH)
 
 # Some definitions

-WAIT_BETWEEN_REQUESTS = timedelta(0, 1)
-SOCKET_TIMEOUT = 4
+WAIT_BETWEEN_REQUESTS = timedelta(0, 2)
+SOCKET_TIMEOUT = 6

 
 
 class SimaEch:
     """EchoNest http client
     """
     root_url = 'http://{host}/api/{version}'.format(**ECH)
 
 
 class SimaEch:
     """EchoNest http client
     """
     root_url = 'http://{host}/api/{version}'.format(**ECH)

-    cache = {}
-    timestamp = datetime.utcnow()

     ratelimit = None
     name = 'EchoNest'
     ratelimit = None
     name = 'EchoNest'

+    cache = False
+    stats = {'etag':0,
+            'ccontrol':0,
+            'minrl':120,
+            'total':0}

 
 

-    def __init__(self, cache=True):
-        self.artist = None
-        self._ressource = None
-        self.current_element = None
-        self.caching = cache
-        purge_cache(self.__class__)
-
-    def _fetch(self, payload):
-        """Use cached elements or proceed http request"""
-        url = Request('GET', self._ressource, params=payload,).prepare().url
-        if url in SimaEch.cache:
-            self.current_element = SimaEch.cache.get(url).elem
-            return
+    def __init__(self):
+        self.controller = CacheController(self.cache)
+
+    def _fetch(self, ressource, payload):
+        """
+        Prepare http request
+        Use cached elements or proceed http request
+        """
+        req = Request('GET', ressource, params=payload,
+                      ).prepare()
+        SimaEch.stats.update(total=SimaEch.stats.get('total')+1)
+        if self.cache:
+            cached_response = self.controller.cached_request(req.url, req.headers)
+            if cached_response:
+                SimaEch.stats.update(ccontrol=SimaEch.stats.get('ccontrol')+1)
+                return cached_response.json()

         try:
         try:

-            self._fetch_ws(payload)
+            return self._fetch_ws(req)

         except Timeout:
             raise WSTimeout('Failed to reach server within {0}s'.format(
                                SOCKET_TIMEOUT))
         except Timeout:
             raise WSTimeout('Failed to reach server within {0}s'.format(
                                SOCKET_TIMEOUT))


@@ -73,28 +80,33 @@ class SimaEch:
             raise WSError(err)
 
     @Throttle(WAIT_BETWEEN_REQUESTS)
             raise WSError(err)
 
     @Throttle(WAIT_BETWEEN_REQUESTS)

-    def _fetch_ws(self, payload):
+    def _fetch_ws(self, prepreq):

         """fetch from web service"""
         """fetch from web service"""

-        req = get(self._ressource, params=payload,
-                            timeout=SOCKET_TIMEOUT)
-        self.__class__.ratelimit = req.headers.get('x-ratelimit-remaining', None)
-        if req.status_code is not 200:
-            raise WSHTTPError('{0.status_code}: {0.reason}'.format(req))
-        self.current_element = req.json()
-        self._controls_answer()
-        if self.caching:
-            SimaEch.cache.update({req.url:
-                                 Cache(self.current_element)})
-
-    def _controls_answer(self):
+        sess = Session()
+        resp = sess.send(prepreq, timeout=SOCKET_TIMEOUT)
+        if resp.status_code == 304:
+            SimaEch.stats.update(etag=SimaEch.stats.get('etag')+1)
+            resp = self.controller.update_cached_response(prepreq, resp)
+        elif resp.status_code != 200:
+            raise WSHTTPError('{0.status_code}: {0.reason}'.format(resp))
+        ans = resp.json()
+        self._controls_answer(ans)
+        SimaEch.ratelimit = resp.headers.get('x-ratelimit-remaining', None)
+        minrl = min(int(SimaEch.ratelimit), SimaEch.stats.get('minrl'))
+        SimaEch.stats.update(minrl=minrl)
+        if self.cache:
+            self.controller.cache_response(resp.request, resp)
+        return ans
+
+    def _controls_answer(self, ans):

         """Controls answer.
         """
         """Controls answer.
         """

-        status = self.current_element.get('response').get('status')
+        status = ans.get('response').get('status')

         code = status.get('code')
         if code is 0:
             return True
         if code is 5:
         code = status.get('code')
         if code is 0:
             return True
         if code is 5:

-            raise WSNotFound('Artist not found: "{0}"'.format(self.artist))
+            raise WSNotFound('Artist not found')

         raise WSError(status.get('message'))
 
     def _forge_payload(self, artist, top=False):
         raise WSError(status.get('message'))
 
     def _forge_payload(self, artist, top=False):


@@ -103,7 +115,6 @@ class SimaEch:
         payload = {'api_key': ECH.get('apikey')}
         if not isinstance(artist, Artist):
             raise TypeError('"{0!r}" not an Artist object'.format(artist))
         payload = {'api_key': ECH.get('apikey')}
         if not isinstance(artist, Artist):
             raise TypeError('"{0!r}" not an Artist object'.format(artist))

-        self.artist = artist

         if artist.mbid:
             payload.update(
                     id='musicbrainz:artist:{0}'.format(artist.mbid))
         if artist.mbid:
             payload.update(
                     id='musicbrainz:artist:{0}'.format(artist.mbid))


@@ -120,17 +131,18 @@ class SimaEch:
                 payload.update(artist=name)
             payload.update(results=100)
             payload.update(sort='song_hotttnesss-desc')
                 payload.update(artist=name)
             payload.update(results=100)
             payload.update(sort='song_hotttnesss-desc')

-        return payload
+        # > hashing the URL into a cache key
+        # return a sorted list of 2-tuple to have consistent cache
+        return sorted(payload.items(), key=lambda param: param[0])

 
     def get_similar(self, artist=None):
         """Fetch similar artists
         """
         payload = self._forge_payload(artist)
         # Construct URL
 
     def get_similar(self, artist=None):
         """Fetch similar artists
         """
         payload = self._forge_payload(artist)
         # Construct URL

-        self._ressource = '{0}/artist/similar'.format(SimaEch.root_url)
-        self._fetch(payload)
-        for art in self.current_element.get('response').get('artists'):
-            artist = {}
+        ressource = '{0}/artist/similar'.format(SimaEch.root_url)
+        ans = self._fetch(ressource, payload)
+        for art in ans.get('response').get('artists'):

             mbid = None
             if 'foreign_ids' in art:
                 for frgnid in art.get('foreign_ids'):
             mbid = None
             if 'foreign_ids' in art:
                 for frgnid in art.get('foreign_ids'):


@@ -144,18 +156,18 @@ class SimaEch:
         """
         payload = self._forge_payload(artist, top=True)
         # Construct URL
         """
         payload = self._forge_payload(artist, top=True)
         # Construct URL

-        self._ressource = '{0}/song/search'.format(SimaEch.root_url)
-        self._fetch(payload)
+        ressource = '{0}/song/search'.format(SimaEch.root_url)
+        ans = self._fetch(ressource, payload)

         titles = list()
         titles = list()

-        artist = {
+        art = {

                 'artist': artist.name,
                 'musicbrainz_artistid': artist.mbid,
                 }
                 'artist': artist.name,
                 'musicbrainz_artistid': artist.mbid,
                 }

-        for song in self.current_element.get('response').get('songs'):
+        for song in ans.get('response').get('songs'):

             title = song.get('title')
             if title not in titles:
                 titles.append(title)
             title = song.get('title')
             if title not in titles:
                 titles.append(title)

-                yield Track(title=title, **artist )
+                yield Track(title=title, **art)

 
 
 # VIM MODLINE
 
 
 # VIM MODLINE