X-Git-Url: https://git.kaliko.me/?a=blobdiff_plain;f=sima%2Flib%2Fhttpcli%2Fcontroller.py;fp=sima%2Flib%2Fhttpcli%2Fcontroller.py;h=0000000000000000000000000000000000000000;hb=71500abd7ef16784d027a8a20aa28b06e8a13a4f;hp=c44789512299399265c9e3c17bcf6c25fd624b65;hpb=bed2018b162e3493dd5a5a7fe5433af2005a7d0a;p=mpd-sima.git diff --git a/sima/lib/httpcli/controller.py b/sima/lib/httpcli/controller.py deleted file mode 100644 index c447895..0000000 --- a/sima/lib/httpcli/controller.py +++ /dev/null @@ -1,213 +0,0 @@ -""" -The httplib2 algorithms ported for use with requests. -""" -import re -import calendar -import time - -from sima.lib.httpcli.cache import DictCache -import email.utils - - -URI = re.compile(r"^(([^:/?#]+):)?(//([^/?#]*))?([^?#]*)(\?([^#]*))?(#(.*))?") - - -def parse_uri(uri): - """Parses a URI using the regex given in Appendix B of RFC 3986. - - (scheme, authority, path, query, fragment) = parse_uri(uri) - """ - groups = URI.match(uri).groups() - return (groups[1], groups[3], groups[4], groups[6], groups[8]) - - -class CacheController(object): - """An interface to see if request should cached or not. - """ - def __init__(self, cache=None, cache_etags=True): - self.cache = cache or DictCache() - self.cache_etags = cache_etags - - def _urlnorm(self, uri): - """Normalize the URL to create a safe key for the cache""" - (scheme, authority, path, query, fragment) = parse_uri(uri) - if not scheme or not authority: - raise Exception("Only absolute URIs are allowed. uri = %s" % uri) - authority = authority.lower() - scheme = scheme.lower() - if not path: - path = "/" - - # Could do syntax based normalization of the URI before - # computing the digest. See Section 6.2.2 of Std 66. - request_uri = query and "?".join([path, query]) or path - scheme = scheme.lower() - defrag_uri = scheme + "://" + authority + request_uri - - return defrag_uri - - def cache_url(self, uri): - return self._urlnorm(uri) - - def parse_cache_control(self, headers): - """ - Parse the cache control headers returning a dictionary with values - for the different directives. - """ - retval = {} - - cc_header = 'cache-control' - if 'Cache-Control' in headers: - cc_header = 'Cache-Control' - - if cc_header in headers: - parts = headers[cc_header].split(',') - parts_with_args = [ - tuple([x.strip().lower() for x in part.split("=", 1)]) - for part in parts if -1 != part.find("=")] - parts_wo_args = [(name.strip().lower(), 1) - for name in parts if -1 == name.find("=")] - retval = dict(parts_with_args + parts_wo_args) - return retval - - def cached_request(self, url, headers): - cache_url = self.cache_url(url) - cc = self.parse_cache_control(headers) - - # non-caching states - no_cache = True if 'no-cache' in cc else False - if 'max-age' in cc and cc['max-age'] == 0: - no_cache = True - - # see if it is in the cache anyways - in_cache = self.cache.get(cache_url) - if no_cache or not in_cache: - return False - - # It is in the cache, so lets see if it is going to be - # fresh enough - resp = self.cache.get(cache_url) - - # Check our Vary header to make sure our request headers match - # up. We don't delete it from the though, we just don't return - # our cached value. - # - # NOTE: Because httplib2 stores raw content, it denotes - # headers that were sent in the original response by - # adding -varied-$name. We don't have to do that b/c we - # are storing the object which has a reference to the - # original request. If that changes, then I'd propose - # using the varied headers in the cache key to avoid the - # situation all together. - if 'vary' in resp.headers: - varied_headers = resp.headers['vary'].replace(' ', '').split(',') - original_headers = resp.request.headers - for header in varied_headers: - # If our headers don't match for the headers listed in - # the vary header, then don't use the cached response - if headers.get(header, None) != original_headers.get(header): - return False - - now = time.time() - date = calendar.timegm( - email.utils.parsedate_tz(resp.headers['date']) - ) - current_age = max(0, now - date) - - # TODO: There is an assumption that the result will be a - # requests response object. This may not be best since we - # could probably avoid instantiating or constructing the - # response until we know we need it. - resp_cc = self.parse_cache_control(resp.headers) - - # determine freshness - freshness_lifetime = 0 - if 'max-age' in resp_cc and resp_cc['max-age'].isdigit(): - freshness_lifetime = int(resp_cc['max-age']) - elif 'expires' in resp.headers: - expires = email.utils.parsedate_tz(resp.headers['expires']) - if expires is not None: - expire_time = calendar.timegm(expires) - date - freshness_lifetime = max(0, expire_time) - - # determine if we are setting freshness limit in the req - if 'max-age' in cc: - try: - freshness_lifetime = int(cc['max-age']) - except ValueError: - freshness_lifetime = 0 - - if 'min-fresh' in cc: - try: - min_fresh = int(cc['min-fresh']) - except ValueError: - min_fresh = 0 - # adjust our current age by our min fresh - current_age += min_fresh - - # see how fresh we actually are - fresh = (freshness_lifetime > current_age) - - if fresh: - # make sure we set the from_cache to true - resp.from_cache = True - return resp - - # we're not fresh. If we don't have an Etag, clear it out - if 'etag' not in resp.headers: - self.cache.delete(cache_url) - - if 'etag' in resp.headers: - headers['If-None-Match'] = resp.headers['ETag'] - - if 'last-modified' in resp.headers: - headers['If-Modified-Since'] = resp.headers['Last-Modified'] - - # return the original handler - return False - - def add_headers(self, url): - resp = self.cache.get(url) - if resp and 'etag' in resp.headers: - return {'If-None-Match': resp.headers['etag']} - return {} - - def cache_response(self, request, resp): - """ - Algorithm for caching requests. - - This assumes a requests Response object. - """ - # From httplib2: Don't cache 206's since we aren't going to - # handle byte range requests - if resp.status_code not in [200, 203]: - return - - cc_req = self.parse_cache_control(request.headers) - cc = self.parse_cache_control(resp.headers) - - cache_url = self.cache_url(request.url) - - # Delete it from the cache if we happen to have it stored there - no_store = cc.get('no-store') or cc_req.get('no-store') - if no_store and self.cache.get(cache_url): - self.cache.delete(cache_url) - - # If we've been given an etag, then keep the response - if self.cache_etags and 'etag' in resp.headers: - self.cache.set(cache_url, resp) - - # Add to the cache if the response headers demand it. If there - # is no date header then we can't do anything about expiring - # the cache. - elif 'date' in resp.headers: - # cache when there is a max-age > 0 - if cc and cc.get('max-age'): - if int(cc['max-age']) > 0: - self.cache.set(cache_url, resp) - - # If the request can expire, it means we should cache it - # in the meantime. - elif 'expires' in resp.headers: - if resp.headers['expires']: - self.cache.set(cache_url, resp)