From df9f0f9dae0f8712f672b4eb7f4211e62f6c2e21 Mon Sep 17 00:00:00 2001 From: kaliko Date: Fri, 21 Feb 2014 23:26:59 +0100 Subject: [PATCH] Improved ETag support, add some stats --- sima/lib/cache.py | 7 ++++++- sima/lib/http.py | 2 -- sima/lib/simaecho.py | 16 +++++++++++----- sima/lib/simafm.py | 4 +++- sima/plugins/internal/echonest.py | 7 +++++++ 5 files changed, 27 insertions(+), 9 deletions(-) diff --git a/sima/lib/cache.py b/sima/lib/cache.py index ebed3fc..d717561 100644 --- a/sima/lib/cache.py +++ b/sima/lib/cache.py @@ -23,7 +23,6 @@ dictionary, which in turns means it is not threadsafe for writing. """ import os -import base64 import codecs from hashlib import md5 @@ -93,3 +92,9 @@ class FileCache: def delete(self, key): if not self.forever: os.remove(self._fn(key)) + + def __iter__(self): + for dirpath, dirnames, filenames in os.walk(self.directory): + for item in filenames: + name = os.path.join(dirpath, item) + yield load(codecs.open(name, 'rb')) diff --git a/sima/lib/http.py b/sima/lib/http.py index 8dbfe0d..c07ad38 100644 --- a/sima/lib/http.py +++ b/sima/lib/http.py @@ -58,8 +58,6 @@ class CacheController(object): if not path: path = "/" - # Order of params might changed - query = ''.join(sorted(query.split('&'))) # Could do syntax based normalization of the URI before # computing the digest. See Section 6.2.2 of Std 66. request_uri = query and "?".join([path, query]) or path diff --git a/sima/lib/simaecho.py b/sima/lib/simaecho.py index 27bbce2..2706513 100644 --- a/sima/lib/simaecho.py +++ b/sima/lib/simaecho.py @@ -50,7 +50,10 @@ class SimaEch: ratelimit = None name = 'EchoNest' cache = False - stats = {'304':0, 'cached':0, 'minrl':'120'} + stats = {'etag':0, + 'ccontrol':0, + 'minrl':120, + 'total':0} def __init__(self): self.controller = CacheController(self.cache) @@ -62,10 +65,11 @@ class SimaEch: """ req = Request('GET', ressource, params=payload, ).prepare() + SimaEch.stats.update(total=SimaEch.stats.get('total')+1) if self.cache: cached_response = self.controller.cached_request(req.url, req.headers) if cached_response: - SimaEch.stat.update(cached=SimaEch.stat.get('cached')+1) + SimaEch.stats.update(ccontrol=SimaEch.stats.get('ccontrol')+1) return cached_response.json() try: return self._fetch_ws(req) @@ -81,14 +85,14 @@ class SimaEch: sess = Session() resp = sess.send(prepreq, timeout=SOCKET_TIMEOUT) if resp.status_code == 304: - SimaEch.stats.update({'304':SimaEch.stats.get('304')+1}) + SimaEch.stats.update(etag=SimaEch.stats.get('etag')+1) resp = self.controller.update_cached_response(prepreq, resp) elif resp.status_code != 200: raise WSHTTPError('{0.status_code}: {0.reason}'.format(resp)) ans = resp.json() self._controls_answer(ans) SimaEch.ratelimit = resp.headers.get('x-ratelimit-remaining', None) - minrl = min(SimaEch.ratelimit, SimaEch.stats.get('minrl')) + minrl = min(int(SimaEch.ratelimit), SimaEch.stats.get('minrl')) SimaEch.stats.update(minrl=minrl) if self.cache: self.controller.cache_response(resp.request, resp) @@ -127,7 +131,9 @@ class SimaEch: payload.update(artist=name) payload.update(results=100) payload.update(sort='song_hotttnesss-desc') - return payload + # > hashing the URL into a cache key + # return a sorted list of 2-tuple to have consistent cache + return sorted(payload.items(), key=lambda param: param[0]) def get_similar(self, artist=None): """Fetch similar artists diff --git a/sima/lib/simafm.py b/sima/lib/simafm.py index f54ab01..cc70cf7 100644 --- a/sima/lib/simafm.py +++ b/sima/lib/simafm.py @@ -120,7 +120,9 @@ class SimaFM: payload.update(results=100) if method == 'track': payload.update(track=track) - return payload + # > hashing the URL into a cache key + # return a sorted list of 2-tuple to have consistent cache + return sorted(payload.items(), key=lambda param: param[0]) def get_similar(self, artist=None): """Fetch similar artists diff --git a/sima/plugins/internal/echonest.py b/sima/plugins/internal/echonest.py index ad3a6f9..1c0b1f7 100644 --- a/sima/plugins/internal/echonest.py +++ b/sima/plugins/internal/echonest.py @@ -43,5 +43,12 @@ class EchoNest(WebService): SimaEch.cache = FileCache(join(vardir, 'http')) self.ws = SimaEch + def callback_playlist(self): + if self.player.state != 'play': + return + msg = 'ETag:{etag:>3d}, Cache-Control:{ccontrol:>3d},' + msg += 'total:{total:>3d}, min(rate-limit):{minrl:>3d}' + self.log.debug(msg.format(**SimaEch.stats)) + # VIM MODLINE # vim: ai ts=4 sw=4 sts=4 expandtab -- 2.39.5