Add ETag support for echonest

diff --git a/sima/lib/http.py b/sima/lib/http.py
index 1040c2a98f949daae3a9e15261136b3e77a0df6a..8dbfe0de804de1b47045d852cb8427ed610f83b6 100644 (file)
--- a/sima/lib/http.py
+++ b/sima/lib/http.py
@@ -58,6 +58,8 @@ class CacheController(object):
         if not path:
             path = "/"
 
+        # Order of params might changed
+        query = ''.join(sorted(query.split('&')))
         # Could do syntax based normalization of the URI before
         # computing the digest. See Section 6.2.2 of Std 66.
         request_uri = query and "?".join([path, query]) or path
@@ -173,11 +175,25 @@ class CacheController(object):
             resp.from_cache = True
             return resp
 
-        # we're not fresh.
-        self.cache.delete(cache_url)
+        # we're not fresh. If we don't have an Etag, clear it out
+        if 'etag' not in resp.headers:
+            self.cache.delete(cache_url)
+
+        if 'etag' in resp.headers:
+            headers['If-None-Match'] = resp.headers['ETag']
+
+        if 'last-modified' in resp.headers:
+            headers['If-Modified-Since'] = resp.headers['Last-Modified']
+
         # return the original handler
         return False
 
+    def add_headers(self, url):
+        resp = self.cache.get(url)
+        if resp and 'etag' in resp.headers:
+            return {'If-None-Match': resp.headers['etag']}
+        return {}
+
     def cache_response(self, request, resp):
         """
         Algorithm for caching requests.
@@ -199,10 +215,14 @@ class CacheController(object):
         if no_store and self.cache.get(cache_url):
             self.cache.delete(cache_url)
 
+        # If we've been given an etag, then keep the response
+        if self.cache_etags and 'etag' in resp.headers:
+            self.cache.set(cache_url, resp)
+
         # Add to the cache if the response headers demand it. If there
         # is no date header then we can't do anything about expiring
         # the cache.
-        if 'date' in resp.headers:
+        elif 'date' in resp.headers:
             # cache when there is a max-age > 0
             if cc_resp and cc_resp.get('max-age'):
                 if int(cc_resp['max-age']) > 0:
@@ -213,3 +233,37 @@ class CacheController(object):
             elif 'expires' in resp.headers:
                 if resp.headers['expires']:
                     self.cache.set(cache_url, resp)
+
+    def update_cached_response(self, request, response):
+        """On a 304 we will get a new set of headers that we want to
+        update our cached value with, assuming we have one.
+
+        This should only ever be called when we've sent an ETag and
+        gotten a 304 as the response.
+        """
+        cache_url = self.cache_url(request.url)
+
+        resp = self.cache.get(cache_url)
+
+        if not resp:
+            # we didn't have a cached response
+            return response
+
+        # did so lets update our headers
+        resp.headers.update(response.headers)
+
+        # we want a 200 b/c we have content via the cache
+        request.status_code = 200
+
+        # update the request as it has the if-none-match header + any
+        # other headers that the server might have updated (ie Date,
+        # Cache-Control, Expires, etc.)
+        resp.request = request
+
+        # update our cache
+        self.cache.set(cache_url, resp)
+
+        # Let everyone know this was from the cache.
+        resp.from_cache = True
+
+        return resp

diff --git a/sima/lib/simaecho.py b/sima/lib/simaecho.py
index 36c985e6d14b5b652b99d69ab10009fe2a850b8e..27bbce2d0f122f8952438aafbf2979538a616f37 100644 (file)
--- a/sima/lib/simaecho.py
+++ b/sima/lib/simaecho.py
@@ -25,7 +25,7 @@ __version__ = '0.0.2'
 __author__ = 'Jack Kaliko'
 
 
-from datetime import datetime, timedelta
+from datetime import timedelta
 
 from requests import Session, Request, Timeout, ConnectionError
 
@@ -50,6 +50,7 @@ class SimaEch:
     ratelimit = None
     name = 'EchoNest'
     cache = False
+    stats = {'304':0, 'cached':0, 'minrl':'120'}
 
     def __init__(self):
         self.controller = CacheController(self.cache)
@@ -64,6 +65,7 @@ class SimaEch:
         if self.cache:
             cached_response = self.controller.cached_request(req.url, req.headers)
             if cached_response:
+                SimaEch.stat.update(cached=SimaEch.stat.get('cached')+1)
                 return cached_response.json()
         try:
             return self._fetch_ws(req)
@@ -78,11 +80,16 @@ class SimaEch:
         """fetch from web service"""
         sess = Session()
         resp = sess.send(prepreq, timeout=SOCKET_TIMEOUT)
-        self.__class__.ratelimit = resp.headers.get('x-ratelimit-remaining', None)
-        if resp.status_code is not 200:
+        if resp.status_code == 304:
+            SimaEch.stats.update({'304':SimaEch.stats.get('304')+1})
+            resp = self.controller.update_cached_response(prepreq, resp)
+        elif resp.status_code != 200:
             raise WSHTTPError('{0.status_code}: {0.reason}'.format(resp))
         ans = resp.json()
         self._controls_answer(ans)
+        SimaEch.ratelimit = resp.headers.get('x-ratelimit-remaining', None)
+        minrl = min(SimaEch.ratelimit, SimaEch.stats.get('minrl'))
+        SimaEch.stats.update(minrl=minrl)
         if self.cache:
             self.controller.cache_response(resp.request, resp)
         return ans
@@ -130,7 +137,6 @@ class SimaEch:
         ressource = '{0}/artist/similar'.format(SimaEch.root_url)
         ans = self._fetch(ressource, payload)
         for art in ans.get('response').get('artists'):
-            artist = {}
             mbid = None
             if 'foreign_ids' in art:
                 for frgnid in art.get('foreign_ids'):
@@ -147,7 +153,7 @@ class SimaEch:
         ressource = '{0}/song/search'.format(SimaEch.root_url)
         ans = self._fetch(ressource, payload)
         titles = list()
-        artist = {
+        art = {
                 'artist': artist.name,
                 'musicbrainz_artistid': artist.mbid,
                 }
@@ -155,7 +161,7 @@ class SimaEch:
             title = song.get('title')
             if title not in titles:
                 titles.append(title)
-                yield Track(title=title, **artist)
+                yield Track(title=title, **art)
 
 
 # VIM MODLINE

diff --git a/sima/lib/simafm.py b/sima/lib/simafm.py
index 5dafd2ce007890d576160b5fd7af63cdd600787c..f54ab015e99a57a01ddded9e58db83a711795ac9 100644 (file)
--- a/sima/lib/simafm.py
+++ b/sima/lib/simafm.py
@@ -25,7 +25,7 @@ __version__ = '0.5.1'
 __author__ = 'Jack Kaliko'
 
 
-from datetime import datetime, timedelta
+from datetime import timedelta
 
 from requests import Session, Request, Timeout, ConnectionError