From bed2018b162e3493dd5a5a7fe5433af2005a7d0a Mon Sep 17 00:00:00 2001 From: kaliko Date: Thu, 20 Feb 2014 20:09:29 +0100 Subject: [PATCH] Persistent HTTP cache based on CacheControl --- sima/lib/httpcli/__init__.py | 0 sima/lib/httpcli/cache.py | 76 ++++++++++++ sima/lib/httpcli/controller.py | 213 +++++++++++++++++++++++++++++++++ sima/lib/httpcli/filelock.py | 80 +++++++++++++ sima/lib/simaecho.py | 68 ++++++----- 5 files changed, 404 insertions(+), 33 deletions(-) create mode 100644 sima/lib/httpcli/__init__.py create mode 100644 sima/lib/httpcli/cache.py create mode 100644 sima/lib/httpcli/controller.py create mode 100644 sima/lib/httpcli/filelock.py diff --git a/sima/lib/httpcli/__init__.py b/sima/lib/httpcli/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/sima/lib/httpcli/cache.py b/sima/lib/httpcli/cache.py new file mode 100644 index 0000000..22c751d --- /dev/null +++ b/sima/lib/httpcli/cache.py @@ -0,0 +1,76 @@ +""" +The cache object API for implementing caches. The default is just a +dictionary, which in turns means it is not threadsafe for writing. +""" + +import os +import base64 +import codecs + +from hashlib import md5 +from pickle import load, dump +from threading import Lock + +from .filelock import FileLock + + +class BaseCache: + + def get(self, key): + raise NotImplemented() + + def set(self, key, value): + raise NotImplemented() + + def delete(self, key): + raise NotImplemented() + + +class DictCache(BaseCache): + + def __init__(self, init_dict=None): + self.lock = Lock() + self.data = init_dict or {} + + def get(self, key): + return self.data.get(key, None) + + def set(self, key, value): + with self.lock: + self.data.update({key: value}) + + def delete(self, key): + with self.lock: + if key in self.data: + self.data.pop(key) + + +class FileCache: + + def __init__(self, directory, forever=False): + self.directory = directory + self.forever = forever + + if not os.path.isdir(self.directory): + os.mkdir(self.directory) + + def encode(self, x): + return md5(x.encode('utf-8')).hexdigest() + + def _fn(self, name): + return os.path.join(self.directory, self.encode(name)) + + def get(self, key): + name = self._fn(key) + if os.path.exists(name): + return load(codecs.open(name, 'rb')) + + def set(self, key, value): + name = self._fn(key) + with FileLock(name): + with codecs.open(name, 'w+b') as fh: + dump(value, fh) + + def delete(self, key): + if not self.forever: + os.remove(self._fn(key)) diff --git a/sima/lib/httpcli/controller.py b/sima/lib/httpcli/controller.py new file mode 100644 index 0000000..c447895 --- /dev/null +++ b/sima/lib/httpcli/controller.py @@ -0,0 +1,213 @@ +""" +The httplib2 algorithms ported for use with requests. +""" +import re +import calendar +import time + +from sima.lib.httpcli.cache import DictCache +import email.utils + + +URI = re.compile(r"^(([^:/?#]+):)?(//([^/?#]*))?([^?#]*)(\?([^#]*))?(#(.*))?") + + +def parse_uri(uri): + """Parses a URI using the regex given in Appendix B of RFC 3986. + + (scheme, authority, path, query, fragment) = parse_uri(uri) + """ + groups = URI.match(uri).groups() + return (groups[1], groups[3], groups[4], groups[6], groups[8]) + + +class CacheController(object): + """An interface to see if request should cached or not. + """ + def __init__(self, cache=None, cache_etags=True): + self.cache = cache or DictCache() + self.cache_etags = cache_etags + + def _urlnorm(self, uri): + """Normalize the URL to create a safe key for the cache""" + (scheme, authority, path, query, fragment) = parse_uri(uri) + if not scheme or not authority: + raise Exception("Only absolute URIs are allowed. uri = %s" % uri) + authority = authority.lower() + scheme = scheme.lower() + if not path: + path = "/" + + # Could do syntax based normalization of the URI before + # computing the digest. See Section 6.2.2 of Std 66. + request_uri = query and "?".join([path, query]) or path + scheme = scheme.lower() + defrag_uri = scheme + "://" + authority + request_uri + + return defrag_uri + + def cache_url(self, uri): + return self._urlnorm(uri) + + def parse_cache_control(self, headers): + """ + Parse the cache control headers returning a dictionary with values + for the different directives. + """ + retval = {} + + cc_header = 'cache-control' + if 'Cache-Control' in headers: + cc_header = 'Cache-Control' + + if cc_header in headers: + parts = headers[cc_header].split(',') + parts_with_args = [ + tuple([x.strip().lower() for x in part.split("=", 1)]) + for part in parts if -1 != part.find("=")] + parts_wo_args = [(name.strip().lower(), 1) + for name in parts if -1 == name.find("=")] + retval = dict(parts_with_args + parts_wo_args) + return retval + + def cached_request(self, url, headers): + cache_url = self.cache_url(url) + cc = self.parse_cache_control(headers) + + # non-caching states + no_cache = True if 'no-cache' in cc else False + if 'max-age' in cc and cc['max-age'] == 0: + no_cache = True + + # see if it is in the cache anyways + in_cache = self.cache.get(cache_url) + if no_cache or not in_cache: + return False + + # It is in the cache, so lets see if it is going to be + # fresh enough + resp = self.cache.get(cache_url) + + # Check our Vary header to make sure our request headers match + # up. We don't delete it from the though, we just don't return + # our cached value. + # + # NOTE: Because httplib2 stores raw content, it denotes + # headers that were sent in the original response by + # adding -varied-$name. We don't have to do that b/c we + # are storing the object which has a reference to the + # original request. If that changes, then I'd propose + # using the varied headers in the cache key to avoid the + # situation all together. + if 'vary' in resp.headers: + varied_headers = resp.headers['vary'].replace(' ', '').split(',') + original_headers = resp.request.headers + for header in varied_headers: + # If our headers don't match for the headers listed in + # the vary header, then don't use the cached response + if headers.get(header, None) != original_headers.get(header): + return False + + now = time.time() + date = calendar.timegm( + email.utils.parsedate_tz(resp.headers['date']) + ) + current_age = max(0, now - date) + + # TODO: There is an assumption that the result will be a + # requests response object. This may not be best since we + # could probably avoid instantiating or constructing the + # response until we know we need it. + resp_cc = self.parse_cache_control(resp.headers) + + # determine freshness + freshness_lifetime = 0 + if 'max-age' in resp_cc and resp_cc['max-age'].isdigit(): + freshness_lifetime = int(resp_cc['max-age']) + elif 'expires' in resp.headers: + expires = email.utils.parsedate_tz(resp.headers['expires']) + if expires is not None: + expire_time = calendar.timegm(expires) - date + freshness_lifetime = max(0, expire_time) + + # determine if we are setting freshness limit in the req + if 'max-age' in cc: + try: + freshness_lifetime = int(cc['max-age']) + except ValueError: + freshness_lifetime = 0 + + if 'min-fresh' in cc: + try: + min_fresh = int(cc['min-fresh']) + except ValueError: + min_fresh = 0 + # adjust our current age by our min fresh + current_age += min_fresh + + # see how fresh we actually are + fresh = (freshness_lifetime > current_age) + + if fresh: + # make sure we set the from_cache to true + resp.from_cache = True + return resp + + # we're not fresh. If we don't have an Etag, clear it out + if 'etag' not in resp.headers: + self.cache.delete(cache_url) + + if 'etag' in resp.headers: + headers['If-None-Match'] = resp.headers['ETag'] + + if 'last-modified' in resp.headers: + headers['If-Modified-Since'] = resp.headers['Last-Modified'] + + # return the original handler + return False + + def add_headers(self, url): + resp = self.cache.get(url) + if resp and 'etag' in resp.headers: + return {'If-None-Match': resp.headers['etag']} + return {} + + def cache_response(self, request, resp): + """ + Algorithm for caching requests. + + This assumes a requests Response object. + """ + # From httplib2: Don't cache 206's since we aren't going to + # handle byte range requests + if resp.status_code not in [200, 203]: + return + + cc_req = self.parse_cache_control(request.headers) + cc = self.parse_cache_control(resp.headers) + + cache_url = self.cache_url(request.url) + + # Delete it from the cache if we happen to have it stored there + no_store = cc.get('no-store') or cc_req.get('no-store') + if no_store and self.cache.get(cache_url): + self.cache.delete(cache_url) + + # If we've been given an etag, then keep the response + if self.cache_etags and 'etag' in resp.headers: + self.cache.set(cache_url, resp) + + # Add to the cache if the response headers demand it. If there + # is no date header then we can't do anything about expiring + # the cache. + elif 'date' in resp.headers: + # cache when there is a max-age > 0 + if cc and cc.get('max-age'): + if int(cc['max-age']) > 0: + self.cache.set(cache_url, resp) + + # If the request can expire, it means we should cache it + # in the meantime. + elif 'expires' in resp.headers: + if resp.headers['expires']: + self.cache.set(cache_url, resp) diff --git a/sima/lib/httpcli/filelock.py b/sima/lib/httpcli/filelock.py new file mode 100644 index 0000000..6dc331b --- /dev/null +++ b/sima/lib/httpcli/filelock.py @@ -0,0 +1,80 @@ +# -*- coding: utf-8 -*- +# https://github.com/dmfrey/FileLock + +import os +import time +import errno + +class FileLockException(Exception): + pass + +class FileLock: + """ A file locking mechanism that has context-manager support so + you can use it in a with statement. This should be relatively cross + compatible as it doesn't rely on msvcrt or fcntl for the locking. + """ + + def __init__(self, file_name, timeout=10, delay=.05): + """ Prepare the file locker. Specify the file to lock and optionally + the maximum timeout and the delay between each attempt to lock. + """ + self.is_locked = False + self.lockfile = os.path.join(os.getcwd(), "%s.lock" % file_name) + self.file_name = file_name + self.timeout = timeout + self.delay = delay + + + def acquire(self): + """ Acquire the lock, if possible. If the lock is in use, it check again + every `wait` seconds. It does this until it either gets the lock or + exceeds `timeout` number of seconds, in which case it throws + an exception. + """ + start_time = time.time() + while True: + try: + self.fd = os.open(self.lockfile, os.O_CREAT|os.O_EXCL|os.O_RDWR) + break; + except OSError as e: + if e.errno != errno.EEXIST: + raise + if (time.time() - start_time) >= self.timeout: + raise FileLockException("Timeout occured.") + time.sleep(self.delay) + self.is_locked = True + + + def release(self): + """ Get rid of the lock by deleting the lockfile. + When working in a `with` statement, this gets automatically + called at the end. + """ + if self.is_locked: + os.close(self.fd) + os.unlink(self.lockfile) + self.is_locked = False + + + def __enter__(self): + """ Activated when used in the with statement. + Should automatically acquire a lock to be used in the with block. + """ + if not self.is_locked: + self.acquire() + return self + + + def __exit__(self, type, value, traceback): + """ Activated at the end of the with statement. + It automatically releases the lock if it isn't locked. + """ + if self.is_locked: + self.release() + + + def __del__(self): + """ Make sure that the FileLock instance doesn't leave a lockfile + lying around. + """ + self.release() diff --git a/sima/lib/simaecho.py b/sima/lib/simaecho.py index bbfc114..592ea03 100644 --- a/sima/lib/simaecho.py +++ b/sima/lib/simaecho.py @@ -21,19 +21,21 @@ Consume EchoNest web service """ -__version__ = '0.0.1' +__version__ = '0.0.2' __author__ = 'Jack Kaliko' from datetime import datetime, timedelta -from requests import get, Request, Timeout, ConnectionError +from requests import Session, Request, Timeout, ConnectionError from sima import ECH from sima.lib.meta import Artist from sima.lib.track import Track +from sima.lib.httpcli.controller import CacheController +from sima.lib.httpcli.cache import FileCache from sima.utils.utils import WSError, WSNotFound, WSTimeout, WSHTTPError -from sima.utils.utils import getws, Throttle, Cache, purge_cache +from sima.utils.utils import getws, Throttle if len(ECH.get('apikey')) == 23: # simple hack allowing imp.reload getws(ECH) @@ -46,26 +48,27 @@ class SimaEch: """EchoNest http client """ root_url = 'http://{host}/api/{version}'.format(**ECH) - cache = {} timestamp = datetime.utcnow() ratelimit = None name = 'EchoNest' + cache = FileCache('/home/kaliko/.local/share/mpd_sima/http') - def __init__(self, cache=True): - self.artist = None + def __init__(self): self._ressource = None self.current_element = None - self.caching = cache - purge_cache(self.__class__) + self.controller = CacheController(self.cache) def _fetch(self, payload): """Use cached elements or proceed http request""" - url = Request('GET', self._ressource, params=payload,).prepare().url - if url in SimaEch.cache: - self.current_element = SimaEch.cache.get(url).elem - return + req = Request('GET', self._ressource, params=payload, + ).prepare() + if self.cache: + cached_response = self.controller.cached_request(req.url, req.headers) + if cached_response: + return cached_response.json() + try: - self._fetch_ws(payload) + return self._fetch_ws(req) except Timeout: raise WSTimeout('Failed to reach server within {0}s'.format( SOCKET_TIMEOUT)) @@ -73,28 +76,28 @@ class SimaEch: raise WSError(err) @Throttle(WAIT_BETWEEN_REQUESTS) - def _fetch_ws(self, payload): + def _fetch_ws(self, prepreq): """fetch from web service""" - req = get(self._ressource, params=payload, - timeout=SOCKET_TIMEOUT) - self.__class__.ratelimit = req.headers.get('x-ratelimit-remaining', None) - if req.status_code is not 200: - raise WSHTTPError('{0.status_code}: {0.reason}'.format(req)) - self.current_element = req.json() - self._controls_answer() - if self.caching: - SimaEch.cache.update({req.url: - Cache(self.current_element)}) - - def _controls_answer(self): + sess = Session() + resp = sess.send(prepreq, timeout=SOCKET_TIMEOUT) + self.__class__.ratelimit = resp.headers.get('x-ratelimit-remaining', None) + if resp.status_code is not 200: + raise WSHTTPError('{0.status_code}: {0.reason}'.format(resp)) + ans = resp.json() + self._controls_answer(ans) + if self.cache: + self.controller.cache_response(resp.request, resp) + return ans + + def _controls_answer(self, ans): """Controls answer. """ - status = self.current_element.get('response').get('status') + status = ans.get('response').get('status') code = status.get('code') if code is 0: return True if code is 5: - raise WSNotFound('Artist not found: "{0}"'.format(self.artist)) + raise WSNotFound('Artist not found') raise WSError(status.get('message')) def _forge_payload(self, artist, top=False): @@ -103,7 +106,6 @@ class SimaEch: payload = {'api_key': ECH.get('apikey')} if not isinstance(artist, Artist): raise TypeError('"{0!r}" not an Artist object'.format(artist)) - self.artist = artist if artist.mbid: payload.update( id='musicbrainz:artist:{0}'.format(artist.mbid)) @@ -128,8 +130,8 @@ class SimaEch: payload = self._forge_payload(artist) # Construct URL self._ressource = '{0}/artist/similar'.format(SimaEch.root_url) - self._fetch(payload) - for art in self.current_element.get('response').get('artists'): + ans = self._fetch(payload) + for art in ans.get('response').get('artists'): artist = {} mbid = None if 'foreign_ids' in art: @@ -145,13 +147,13 @@ class SimaEch: payload = self._forge_payload(artist, top=True) # Construct URL self._ressource = '{0}/song/search'.format(SimaEch.root_url) - self._fetch(payload) + ans = self._fetch(payload) titles = list() artist = { 'artist': artist.name, 'musicbrainz_artistid': artist.mbid, } - for song in self.current_element.get('response').get('songs'): + for song in ans.get('response').get('songs'): title = song.get('title') if title not in titles: titles.append(title) -- 2.39.5