Reuse http connections

[mpd-sima.git] / sima / lib / http.py

diff --git a/sima/lib/http.py b/sima/lib/http.py
index 8dbfe0de804de1b47045d852cb8427ed610f83b6..d66b6b793a182fb8abc1191de48995f23621b2d0 100644 (file)
--- a/sima/lib/http.py
+++ b/sima/lib/http.py
@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 
-# Copyright (c) 2014 Jack Kaliko <kaliko@azylum.org>
+# Copyright (c) 2014-2015 Jack Kaliko <kaliko@azylum.org>
 # Copyright (c) 2012, 2013 Eric Larson <eric@ionrock.org>
 #
 #   This program is free software: you can redistribute it and/or modify
@@ -26,6 +26,10 @@ import time
 
 import email.utils
 
+from requests import Session, Request, Timeout, ConnectionError
+
+from sima import SOCKET_TIMEOUT, WAIT_BETWEEN_REQUESTS
+from sima.utils.utils import WSError, WSTimeout, WSHTTPError, Throttle
 from .cache import DictCache
 
 
@@ -44,6 +48,8 @@ def parse_uri(uri):
 class CacheController(object):
     """An interface to see if request should cached or not.
     """
+    CACHE_ANYWAY = False
+
     def __init__(self, cache=None, cache_etags=True):
         self.cache = cache or DictCache()
         self.cache_etags = cache_etags
@@ -58,8 +64,6 @@ class CacheController(object):
         if not path:
             path = "/"
 
-        # Order of params might changed
-        query = ''.join(sorted(query.split('&')))
         # Could do syntax based normalization of the URI before
         # computing the digest. See Section 6.2.2 of Std 66.
         request_uri = query and "?".join([path, query]) or path
@@ -90,17 +94,16 @@ class CacheController(object):
             retval = dict(parts_with_args + parts_wo_args)
         return retval
 
-    def cached_request(self, url, headers):
+    def cached_request(self, request):
         """Return the cached resquest if available and fresh
         """
-        cache_url = self.cache_url(url)
-        cc = self.parse_cache_control(headers)
+        cache_url = self.cache_url(request.url)
+        cc = self.parse_cache_control(request.headers)
 
         # non-caching states
         no_cache = True if 'no-cache' in cc else False
         if 'max-age' in cc and cc['max-age'] == 0:
             no_cache = True
-
         # see if it is in the cache anyways
         in_cache = self.cache.get(cache_url)
         if no_cache or not in_cache:
@@ -127,7 +130,7 @@ class CacheController(object):
             for header in varied_headers:
                 # If our headers don't match for the headers listed in
                 # the vary header, then don't use the cached response
-                if headers.get(header, None) != original_headers.get(header):
+                if request.headers.get(header, None) != original_headers.get(header):
                     return False
 
         now = time.time()
@@ -180,20 +183,14 @@ class CacheController(object):
             self.cache.delete(cache_url)
 
         if 'etag' in resp.headers:
-            headers['If-None-Match'] = resp.headers['ETag']
+            request.headers['If-None-Match'] = resp.headers['ETag']
 
         if 'last-modified' in resp.headers:
-            headers['If-Modified-Since'] = resp.headers['Last-Modified']
+            request.headers['If-Modified-Since'] = resp.headers['Last-Modified']
 
         # return the original handler
         return False
 
-    def add_headers(self, url):
-        resp = self.cache.get(url)
-        if resp and 'etag' in resp.headers:
-            return {'If-None-Match': resp.headers['etag']}
-        return {}
-
     def cache_response(self, request, resp):
         """
         Algorithm for caching requests.
@@ -233,6 +230,12 @@ class CacheController(object):
             elif 'expires' in resp.headers:
                 if resp.headers['expires']:
                     self.cache.set(cache_url, resp)
+            # Force one month max age if no Cache-Control header is found
+            # Overriding header disappearance on LastFM web service...
+            # https://getsatisfaction.com/lastfm/topics/-web-api-http-cache-control-header
+            elif CacheController.CACHE_ANYWAY:
+                resp.headers['Cache-Control'] = 'max-age=2419200'
+                self.cache.set(cache_url, resp)
 
     def update_cached_response(self, request, response):
         """On a 304 we will get a new set of headers that we want to
@@ -267,3 +270,47 @@ class CacheController(object):
         resp.from_cache = True
 
         return resp
+
+
+class HttpClient:
+    def __init__(self, cache=None, stats=None):
+        """
+        Prepare http request
+        Use cached elements or proceed http request
+        """
+        self.stats = stats
+        self.controller = CacheController(cache)
+        self.sess = Session()
+
+    def __call__(self, ress, payload):
+        req = Request('GET', ress, params=payload,).prepare()
+        if self.stats:
+            self.stats.update(total=self.stats.get('total')+1)
+        cached_response = self.controller.cached_request(req)
+        if cached_response:
+            if self.stats:
+                self.stats.update(ccontrol=self.stats.get('ccontrol')+1)
+            return cached_response
+        try:
+            return self.fetch_ws(req)
+        except Timeout:
+            raise WSTimeout('Failed to reach server within {0}s'.format(
+                SOCKET_TIMEOUT))
+        except ConnectionError as err:
+            raise WSError(err)
+
+    @Throttle(WAIT_BETWEEN_REQUESTS)
+    def fetch_ws(self, prepreq):
+        """fetch from web service"""
+        settings = self.sess.merge_environment_settings(prepreq.url, {}, None, False, None)
+        resp = self.sess.send(prepreq, timeout=SOCKET_TIMEOUT, **settings)
+        if resp.status_code == 304:
+            self.stats.update(etag=self.stats.get('etag')+1)
+            resp = self.controller.update_cached_response(prepreq, resp)
+        elif resp.status_code != 200:
+            raise WSHTTPError('{0.status_code}: {0.reason}'.format(resp))
+        self.controller.cache_response(resp.request, resp)
+        return resp
+
+# VIM MODLINE
+# vim: ai ts=4 sw=4 sts=4 expandtab