# -*- coding: utf-8 -*-
-# Copyright (c) 2014-2015 Jack Kaliko <kaliko@azylum.org>
+# Copyright (c) 2014-2015, 2020 kaliko <kaliko@azylum.org>
# Copyright (c) 2012, 2013 Eric Larson <eric@ionrock.org>
#
# This program is free software: you can redistribute it and/or modify
class CacheController(object):
"""An interface to see if request should cached or not.
"""
+ CACHE_ANYWAY = False
+
def __init__(self, cache=None, cache_etags=True):
self.cache = cache or DictCache()
self.cache_etags = cache_etags
no_cache = True if 'no-cache' in cc else False
if 'max-age' in cc and cc['max-age'] == 0:
no_cache = True
-
# see if it is in the cache anyways
in_cache = self.cache.get(cache_url)
if no_cache or not in_cache:
# return the original handler
return False
- def add_headers(self, url):
- resp = self.cache.get(url)
- if resp and 'etag' in resp.headers:
- return {'If-None-Match': resp.headers['etag']}
- return {}
-
def cache_response(self, request, resp):
"""
Algorithm for caching requests.
elif 'expires' in resp.headers:
if resp.headers['expires']:
self.cache.set(cache_url, resp)
+ # Force one month max age if no Cache-Control header is found
+ # Overriding header disappearance on LastFM web service...
+ # https://getsatisfaction.com/lastfm/topics/-web-api-http-cache-control-header
+ elif CacheController.CACHE_ANYWAY:
+ resp.headers['Cache-Control'] = 'max-age=2419200'
+ self.cache.set(cache_url, resp)
def update_cached_response(self, request, response):
"""On a 304 we will get a new set of headers that we want to
"""
self.stats = stats
self.controller = CacheController(cache)
+ self.sess = Session()
def __call__(self, ress, payload):
req = Request('GET', ress, params=payload,).prepare()
@Throttle(WAIT_BETWEEN_REQUESTS)
def fetch_ws(self, prepreq):
"""fetch from web service"""
- sess = Session()
- settings = sess.merge_environment_settings(prepreq.url, {}, None, False, None)
- resp = sess.send(prepreq, timeout=SOCKET_TIMEOUT, **settings)
+ settings = self.sess.merge_environment_settings(prepreq.url, {}, None, False, None)
+ resp = self.sess.send(prepreq, timeout=SOCKET_TIMEOUT, **settings)
if resp.status_code == 304:
self.stats.update(etag=self.stats.get('etag')+1)
resp = self.controller.update_cached_response(prepreq, resp)
elif resp.status_code != 200:
raise WSHTTPError('{0.status_code}: {0.reason}'.format(resp))
- ratelimit = resp.headers.get('x-ratelimit-remaining', None)
- if ratelimit and self.stats:
- minrl = min(int(ratelimit), self.stats.get('minrl'))
- self.stats.update(minrl=minrl)
self.controller.cache_response(resp.request, resp)
return resp