+++ /dev/null
-"""
-The httplib2 algorithms ported for use with requests.
-"""
-import re
-import calendar
-import time
-
-from sima.lib.httpcli.cache import DictCache
-import email.utils
-
-
-URI = re.compile(r"^(([^:/?#]+):)?(//([^/?#]*))?([^?#]*)(\?([^#]*))?(#(.*))?")
-
-
-def parse_uri(uri):
- """Parses a URI using the regex given in Appendix B of RFC 3986.
-
- (scheme, authority, path, query, fragment) = parse_uri(uri)
- """
- groups = URI.match(uri).groups()
- return (groups[1], groups[3], groups[4], groups[6], groups[8])
-
-
-class CacheController(object):
- """An interface to see if request should cached or not.
- """
- def __init__(self, cache=None, cache_etags=True):
- self.cache = cache or DictCache()
- self.cache_etags = cache_etags
-
- def _urlnorm(self, uri):
- """Normalize the URL to create a safe key for the cache"""
- (scheme, authority, path, query, fragment) = parse_uri(uri)
- if not scheme or not authority:
- raise Exception("Only absolute URIs are allowed. uri = %s" % uri)
- authority = authority.lower()
- scheme = scheme.lower()
- if not path:
- path = "/"
-
- # Could do syntax based normalization of the URI before
- # computing the digest. See Section 6.2.2 of Std 66.
- request_uri = query and "?".join([path, query]) or path
- scheme = scheme.lower()
- defrag_uri = scheme + "://" + authority + request_uri
-
- return defrag_uri
-
- def cache_url(self, uri):
- return self._urlnorm(uri)
-
- def parse_cache_control(self, headers):
- """
- Parse the cache control headers returning a dictionary with values
- for the different directives.
- """
- retval = {}
-
- cc_header = 'cache-control'
- if 'Cache-Control' in headers:
- cc_header = 'Cache-Control'
-
- if cc_header in headers:
- parts = headers[cc_header].split(',')
- parts_with_args = [
- tuple([x.strip().lower() for x in part.split("=", 1)])
- for part in parts if -1 != part.find("=")]
- parts_wo_args = [(name.strip().lower(), 1)
- for name in parts if -1 == name.find("=")]
- retval = dict(parts_with_args + parts_wo_args)
- return retval
-
- def cached_request(self, url, headers):
- cache_url = self.cache_url(url)
- cc = self.parse_cache_control(headers)
-
- # non-caching states
- no_cache = True if 'no-cache' in cc else False
- if 'max-age' in cc and cc['max-age'] == 0:
- no_cache = True
-
- # see if it is in the cache anyways
- in_cache = self.cache.get(cache_url)
- if no_cache or not in_cache:
- return False
-
- # It is in the cache, so lets see if it is going to be
- # fresh enough
- resp = self.cache.get(cache_url)
-
- # Check our Vary header to make sure our request headers match
- # up. We don't delete it from the though, we just don't return
- # our cached value.
- #
- # NOTE: Because httplib2 stores raw content, it denotes
- # headers that were sent in the original response by
- # adding -varied-$name. We don't have to do that b/c we
- # are storing the object which has a reference to the
- # original request. If that changes, then I'd propose
- # using the varied headers in the cache key to avoid the
- # situation all together.
- if 'vary' in resp.headers:
- varied_headers = resp.headers['vary'].replace(' ', '').split(',')
- original_headers = resp.request.headers
- for header in varied_headers:
- # If our headers don't match for the headers listed in
- # the vary header, then don't use the cached response
- if headers.get(header, None) != original_headers.get(header):
- return False
-
- now = time.time()
- date = calendar.timegm(
- email.utils.parsedate_tz(resp.headers['date'])
- )
- current_age = max(0, now - date)
-
- # TODO: There is an assumption that the result will be a
- # requests response object. This may not be best since we
- # could probably avoid instantiating or constructing the
- # response until we know we need it.
- resp_cc = self.parse_cache_control(resp.headers)
-
- # determine freshness
- freshness_lifetime = 0
- if 'max-age' in resp_cc and resp_cc['max-age'].isdigit():
- freshness_lifetime = int(resp_cc['max-age'])
- elif 'expires' in resp.headers:
- expires = email.utils.parsedate_tz(resp.headers['expires'])
- if expires is not None:
- expire_time = calendar.timegm(expires) - date
- freshness_lifetime = max(0, expire_time)
-
- # determine if we are setting freshness limit in the req
- if 'max-age' in cc:
- try:
- freshness_lifetime = int(cc['max-age'])
- except ValueError:
- freshness_lifetime = 0
-
- if 'min-fresh' in cc:
- try:
- min_fresh = int(cc['min-fresh'])
- except ValueError:
- min_fresh = 0
- # adjust our current age by our min fresh
- current_age += min_fresh
-
- # see how fresh we actually are
- fresh = (freshness_lifetime > current_age)
-
- if fresh:
- # make sure we set the from_cache to true
- resp.from_cache = True
- return resp
-
- # we're not fresh. If we don't have an Etag, clear it out
- if 'etag' not in resp.headers:
- self.cache.delete(cache_url)
-
- if 'etag' in resp.headers:
- headers['If-None-Match'] = resp.headers['ETag']
-
- if 'last-modified' in resp.headers:
- headers['If-Modified-Since'] = resp.headers['Last-Modified']
-
- # return the original handler
- return False
-
- def add_headers(self, url):
- resp = self.cache.get(url)
- if resp and 'etag' in resp.headers:
- return {'If-None-Match': resp.headers['etag']}
- return {}
-
- def cache_response(self, request, resp):
- """
- Algorithm for caching requests.
-
- This assumes a requests Response object.
- """
- # From httplib2: Don't cache 206's since we aren't going to
- # handle byte range requests
- if resp.status_code not in [200, 203]:
- return
-
- cc_req = self.parse_cache_control(request.headers)
- cc = self.parse_cache_control(resp.headers)
-
- cache_url = self.cache_url(request.url)
-
- # Delete it from the cache if we happen to have it stored there
- no_store = cc.get('no-store') or cc_req.get('no-store')
- if no_store and self.cache.get(cache_url):
- self.cache.delete(cache_url)
-
- # If we've been given an etag, then keep the response
- if self.cache_etags and 'etag' in resp.headers:
- self.cache.set(cache_url, resp)
-
- # Add to the cache if the response headers demand it. If there
- # is no date header then we can't do anything about expiring
- # the cache.
- elif 'date' in resp.headers:
- # cache when there is a max-age > 0
- if cc and cc.get('max-age'):
- if int(cc['max-age']) > 0:
- self.cache.set(cache_url, resp)
-
- # If the request can expire, it means we should cache it
- # in the meantime.
- elif 'expires' in resp.headers:
- if resp.headers['expires']:
- self.cache.set(cache_url, resp)
home / dev / repositories / mpd-sima.git / blobdiff