2 The httplib2 algorithms ported for use with requests.
8 from sima.lib.httpcli.cache import DictCache
12 URI = re.compile(r"^(([^:/?#]+):)?(//([^/?#]*))?([^?#]*)(\?([^#]*))?(#(.*))?")
16 """Parses a URI using the regex given in Appendix B of RFC 3986.
18 (scheme, authority, path, query, fragment) = parse_uri(uri)
20 groups = URI.match(uri).groups()
21 return (groups[1], groups[3], groups[4], groups[6], groups[8])
24 class CacheController(object):
25 """An interface to see if request should cached or not.
27 def __init__(self, cache=None, cache_etags=True):
28 self.cache = cache or DictCache()
29 self.cache_etags = cache_etags
31 def _urlnorm(self, uri):
32 """Normalize the URL to create a safe key for the cache"""
33 (scheme, authority, path, query, fragment) = parse_uri(uri)
34 if not scheme or not authority:
35 raise Exception("Only absolute URIs are allowed. uri = %s" % uri)
36 authority = authority.lower()
37 scheme = scheme.lower()
41 # Could do syntax based normalization of the URI before
42 # computing the digest. See Section 6.2.2 of Std 66.
43 request_uri = query and "?".join([path, query]) or path
44 scheme = scheme.lower()
45 defrag_uri = scheme + "://" + authority + request_uri
49 def cache_url(self, uri):
50 return self._urlnorm(uri)
52 def parse_cache_control(self, headers):
54 Parse the cache control headers returning a dictionary with values
55 for the different directives.
59 cc_header = 'cache-control'
60 if 'Cache-Control' in headers:
61 cc_header = 'Cache-Control'
63 if cc_header in headers:
64 parts = headers[cc_header].split(',')
66 tuple([x.strip().lower() for x in part.split("=", 1)])
67 for part in parts if -1 != part.find("=")]
68 parts_wo_args = [(name.strip().lower(), 1)
69 for name in parts if -1 == name.find("=")]
70 retval = dict(parts_with_args + parts_wo_args)
73 def cached_request(self, url, headers):
74 cache_url = self.cache_url(url)
75 cc = self.parse_cache_control(headers)
78 no_cache = True if 'no-cache' in cc else False
79 if 'max-age' in cc and cc['max-age'] == 0:
82 # see if it is in the cache anyways
83 in_cache = self.cache.get(cache_url)
84 if no_cache or not in_cache:
87 # It is in the cache, so lets see if it is going to be
89 resp = self.cache.get(cache_url)
91 # Check our Vary header to make sure our request headers match
92 # up. We don't delete it from the though, we just don't return
95 # NOTE: Because httplib2 stores raw content, it denotes
96 # headers that were sent in the original response by
97 # adding -varied-$name. We don't have to do that b/c we
98 # are storing the object which has a reference to the
99 # original request. If that changes, then I'd propose
100 # using the varied headers in the cache key to avoid the
101 # situation all together.
102 if 'vary' in resp.headers:
103 varied_headers = resp.headers['vary'].replace(' ', '').split(',')
104 original_headers = resp.request.headers
105 for header in varied_headers:
106 # If our headers don't match for the headers listed in
107 # the vary header, then don't use the cached response
108 if headers.get(header, None) != original_headers.get(header):
112 date = calendar.timegm(
113 email.utils.parsedate_tz(resp.headers['date'])
115 current_age = max(0, now - date)
117 # TODO: There is an assumption that the result will be a
118 # requests response object. This may not be best since we
119 # could probably avoid instantiating or constructing the
120 # response until we know we need it.
121 resp_cc = self.parse_cache_control(resp.headers)
123 # determine freshness
124 freshness_lifetime = 0
125 if 'max-age' in resp_cc and resp_cc['max-age'].isdigit():
126 freshness_lifetime = int(resp_cc['max-age'])
127 elif 'expires' in resp.headers:
128 expires = email.utils.parsedate_tz(resp.headers['expires'])
129 if expires is not None:
130 expire_time = calendar.timegm(expires) - date
131 freshness_lifetime = max(0, expire_time)
133 # determine if we are setting freshness limit in the req
136 freshness_lifetime = int(cc['max-age'])
138 freshness_lifetime = 0
140 if 'min-fresh' in cc:
142 min_fresh = int(cc['min-fresh'])
145 # adjust our current age by our min fresh
146 current_age += min_fresh
148 # see how fresh we actually are
149 fresh = (freshness_lifetime > current_age)
152 # make sure we set the from_cache to true
153 resp.from_cache = True
156 # we're not fresh. If we don't have an Etag, clear it out
157 if 'etag' not in resp.headers:
158 self.cache.delete(cache_url)
160 if 'etag' in resp.headers:
161 headers['If-None-Match'] = resp.headers['ETag']
163 if 'last-modified' in resp.headers:
164 headers['If-Modified-Since'] = resp.headers['Last-Modified']
166 # return the original handler
169 def add_headers(self, url):
170 resp = self.cache.get(url)
171 if resp and 'etag' in resp.headers:
172 return {'If-None-Match': resp.headers['etag']}
175 def cache_response(self, request, resp):
177 Algorithm for caching requests.
179 This assumes a requests Response object.
181 # From httplib2: Don't cache 206's since we aren't going to
182 # handle byte range requests
183 if resp.status_code not in [200, 203]:
186 cc_req = self.parse_cache_control(request.headers)
187 cc = self.parse_cache_control(resp.headers)
189 cache_url = self.cache_url(request.url)
191 # Delete it from the cache if we happen to have it stored there
192 no_store = cc.get('no-store') or cc_req.get('no-store')
193 if no_store and self.cache.get(cache_url):
194 self.cache.delete(cache_url)
196 # If we've been given an etag, then keep the response
197 if self.cache_etags and 'etag' in resp.headers:
198 self.cache.set(cache_url, resp)
200 # Add to the cache if the response headers demand it. If there
201 # is no date header then we can't do anything about expiring
203 elif 'date' in resp.headers:
204 # cache when there is a max-age > 0
205 if cc and cc.get('max-age'):
206 if int(cc['max-age']) > 0:
207 self.cache.set(cache_url, resp)
209 # If the request can expire, it means we should cache it
211 elif 'expires' in resp.headers:
212 if resp.headers['expires']:
213 self.cache.set(cache_url, resp)