Ticket #5126: caching_agent.patch
File caching_agent.patch, 13.4 KB (added by , 10 years ago) |
---|
-
web/test/test_webclient.py
1834 1834 1835 1835 return deferred.addCallback(checkFailure) 1836 1836 1837 class CachingAgentTests(unittest.TestCase, 1838 FakeReactorAndConnectMixin): 1839 1840 def setUp(self): 1841 """ 1842 Create an L{Agent} wrapped around a fake reactor with a memory cache as a backend. 1843 """ 1844 self.reactor = self.Reactor() 1845 agent = client.Agent(self.reactor) 1846 agent._connect = self._dummyConnect 1847 self.cache = client.MemoryCache() 1848 self.agent = client.CachingAgent( 1849 agent,cache=self.cache) 1850 1851 def test_requestHeaders(self): 1852 1853 e = {"etag":"qwertz", 1854 "last-modified":"Sun, 06 Nov 1994 08:49:37 GMT", 1855 "content": "0123456789"} 1856 self.cache.put("http://example.com/foo",e) 1857 1858 self.agent.request('GET','http://example.com/foo') 1859 1860 protocol = self.protocol 1861 1862 self.assertEquals(len(protocol.requests),1) 1863 req,res = protocol.requests.pop() 1864 1865 self.assertEquals(req.headers.getRawHeaders("if-none-match"), 1866 [e["etag"]]) 1867 self.assertEquals(req.headers.getRawHeaders("if-modified-since"), 1868 [e["last-modified"]]) 1869 1870 def test_freshContent(self): 1871 1872 d = self.agent.request('GET','http://example.com/foo') 1873 1874 req,res = self.protocol.requests.pop() 1875 1876 headers = http_headers.Headers({'etag': ['qwertz'], 1877 "last-modified": ["Sun, 06 Nov 1994 08:49:37 GMT"]}) 1878 1879 data = "0123456789" 1880 transport = StringTransport() 1881 response = Response(('HTTP',1,1),200,'OK',headers,transport) 1882 response.length = 10 1883 res.callback(response) 1884 1885 def checkResponse(result): 1886 self.assertNotIdentical(result,response) 1887 self.assertEquals(result.version,('HTTP',1,1)) 1888 self.assertEquals(result.code,200) 1889 self.assertEquals(result.phrase,'OK') 1890 self.assertEquals(result.headers.getRawHeaders("etag"),["qwertz"]) 1891 self.assertEquals(result.headers.getRawHeaders("last-modified"), 1892 ["Sun, 06 Nov 1994 08:49:37 GMT"]) 1893 1894 1895 response._bodyDataReceived(data) 1896 response._bodyDataFinished() 1897 1898 protocol = SimpleAgentProtocol() 1899 result.deliverBody(protocol) 1900 1901 self.assertEquals(protocol.received,[data]) 1902 1903 c = self.cache.get('http://example.com/foo') 1904 self.assertEquals(c["content"],data) 1905 self.assertEquals(c["etag"],"qwertz") 1906 self.assertEquals(c["last-modified"],"Sun, 06 Nov 1994 08:49:37 GMT") 1907 1908 1909 return defer.gatherResults([protocol.made,protocol.finished]) 1910 1911 d.addCallback(checkResponse) 1912 1913 return d 1914 1915 1916 def test_cachedContent(self): 1917 1918 data = "0123456789" 1919 1920 e = {"etag":"qwertz", 1921 "last-modified":"Sun, 06 Nov 1994 08:49:37 GMT", 1922 "content": data} 1923 1924 self.cache.put("http://example.com/foo",e) 1925 1926 d = self.agent.request('GET','http://example.com/foo') 1927 1928 req,res = self.protocol.requests.pop() 1929 1930 headers = http_headers.Headers({'etag': ['qwertz'], 1931 "last-modified": ["Sun, 06 Nov 1994 08:49:37 GMT"]}) 1932 1933 transport = StringTransport() 1934 response = Response(('HTTP',1,1),304,'OK',headers,transport) 1935 response.length = 10 1936 res.callback(response) 1937 1938 def checkResponse(result): 1939 1940 self.assertNotIdentical(result,response) 1941 self.assertEquals(result.version,('HTTP',1,1)) 1942 self.assertEquals(result.code,200) 1943 self.assertEquals(result.phrase,'OK') 1944 self.assertEquals(result.headers.getRawHeaders("etag"),["qwertz"]) 1945 self.assertEquals(result.headers.getRawHeaders("last-modified"), 1946 ["Sun, 06 Nov 1994 08:49:37 GMT"]) 1947 1948 response._bodyDataReceived("") 1949 response._bodyDataFinished() 1950 1951 protocol = SimpleAgentProtocol() 1952 result.deliverBody(protocol) 1953 1954 self.assertEquals(protocol.received,[data]) 1955 1956 c = self.cache.get('http://example.com/foo') 1957 self.assertEquals(c["content"],data) 1958 self.assertEquals(c["etag"],"qwertz") 1959 self.assertEquals(c["last-modified"],"Sun, 06 Nov 1994 08:49:37 GMT") 1960 1961 return defer.gatherResults([protocol.made,protocol.finished]) 1962 1963 d.addCallback(checkResponse) 1837 1964 1965 return d 1838 1966 1839 1967 if ssl is None or not hasattr(ssl, 'DefaultOpenSSLContextFactory'): 1840 1968 for case in [WebClientSSLTestCase, WebClientRedirectBetweenSSLandPlainText]: -
web/client.py
10 10 from urlparse import urlunparse 11 11 import zlib 12 12 13 from zope.interface import implements 14 13 15 from twisted.python import log 14 16 from twisted.web import http 15 17 from twisted.internet import defer, protocol, reactor … … 18 20 from twisted.python.util import InsensitiveDict 19 21 from twisted.python.components import proxyForInterface 20 22 from twisted.web import error 21 from twisted.web.iweb import UNKNOWN_LENGTH, IResponse 23 from twisted.web.iweb import UNKNOWN_LENGTH, IResponse, IHTTPCache 22 24 from twisted.web.http_headers import Headers 23 25 from twisted.python.compat import set 24 26 … … 1003 1005 1004 1006 1005 1007 1008 class CacheBodyProducer(proxyForInterface(IResponse)): 1009 """ 1010 A wrapper for a L{Response} instance which handles cached response bodies. 1011 This type of response will be used if a cache hit occurs. 1012 1013 @ivar original: The original L{Response} object. 1014 1015 @since: 11.1 1016 """ 1017 1018 def __init__(self,response,cache): 1019 self.original = response 1020 self.cache = cache 1021 self.length = len(cache["content"]) 1022 1023 1024 def deliverBody(self,protocol): 1025 """ 1026 Override C{deliverBody} to deliver the cached content 1027 to the given protocol 1028 """ 1029 try: 1030 protocol.connectionMade() 1031 protocol.dataReceived(self.cache["content"]) 1032 protocol.connectionLost(failure.Failure(ResponseDone("Body delivered from cache."))) 1033 except: 1034 protocol.connectionLost(failure.Failure()) 1035 1036 1037 1038 class CacheBodyUpdater(proxyForInterface(IResponse)): 1039 """ 1040 A wrapper for a L{Response} instance which transparently generates a cache entry 1041 for new content. 1042 This type of response will be used if a cache muss occurs. 1043 1044 @ivar original: The original L{Response} object. 1045 1046 @since: 11.1 1047 """ 1048 def __init__(self,response,cache,cacheKey): 1049 self.original = response 1050 self.cache = cache 1051 self.cacheKey = cacheKey 1052 1053 1054 def deliverBody(self,protocol): 1055 self.original.deliverBody(_CachingProtocol(protocol,self.cache,self.cacheKey)) 1056 1057 1058 1059 class _CachingProtocol(proxyForInterface(IProtocol)): 1060 """ 1061 A L{Protocol} implementation which wraps another one, transparently 1062 cacheing the content as data is received. 1063 1064 @ivar cache: The cache object used for storing cached responses. 1065 1066 @ivar cacheKey: The key to identify the current response by. 1067 1068 @since: 11.1 1069 """ 1070 1071 def __init__(self,protocol,cache,cacheKey): 1072 self.original = protocol 1073 self.cache = cache 1074 self.cacheKey = cacheKey 1075 self.buffer = "" 1076 1077 1078 def dataReceived(self,data): 1079 """ 1080 Buffer all incoming C{data} before writing it to the receiving protocol 1081 """ 1082 self.buffer += data 1083 self.original.dataReceived(data) 1084 1085 1086 def connectionLost(self,reason): 1087 """ 1088 Forward the connection lost event, placing the buffered content into 1089 the cache beforehand. 1090 """ 1091 entry = self.cache.get(self.cacheKey) 1092 if entry is None: 1093 entry = {} 1094 entry["content"] = self.buffer 1095 self.cache.put(self.cacheKey,entry) 1096 self.original.connectionLost(reason) 1097 1098 1099 1100 class MemoryCache(object): 1101 """ 1102 An L{IHTTPCache} storing all data in system memory. 1103 A cache entry for this data store musst be a C{dict} object the contains all 1104 nessecary http header fileds as keys plus an extra 'content' key to map the 1105 request body. 1106 1107 @ivar _storage: The C{dict} storing the cache entries. 1108 """ 1109 implements(IHTTPCache) 1110 1111 def __init__(self): 1112 self._storage = {} 1113 1114 1115 def get(self,key,default=None): 1116 """ 1117 Returns a cache entry from the cache if one exists for a specific C{key}. 1118 If none exists, C{default} is returned. 1119 """ 1120 return self._storage.get(key,default) 1121 1122 1123 def put(self,key,entry): 1124 """ 1125 Place a cache C{entry} into the store referenced by a unique C{key}. 1126 If an entry already exists for a key, this entry will be overwritten. 1127 """ 1128 self._storage[key] = entry 1129 1130 1131 def delete(self,key): 1132 """ 1133 Delete all entries from the cache that are referenced by C{key}. 1134 """ 1135 if key in self._storage: 1136 del self._storage[key] 1137 1138 1139 1140 class CachingAgent(object): 1141 """ 1142 An L{Agent} wrapper to handle cachable content. 1143 1144 I manages a cache system by looking at certain http headers and determains 1145 if it sould satisfy a request with localy cached content or if a fresh copy 1146 should be used. 1147 Currently, the following caching-related headers are supported: 1148 etag, last-modified, if-match, if-not-match 1149 1150 @param cache: An instance of a cache to store data and to satisfy responses from. 1151 1152 @since: 11.1 1153 """ 1154 1155 def __init__(self,agent,cache=MemoryCache()): 1156 self._agent = agent 1157 self._cache = cache 1158 1159 1160 def request(self,method,uri,headers=None,bodyProducer=None): 1161 """ 1162 Send a client request which will be checked against the cache. 1163 1164 @see: L{Agent.request}. 1165 """ 1166 if headers is None: 1167 headers = Headers() 1168 else: 1169 headers = headers.copy() 1170 1171 cacheKey = uri 1172 entry = self._cache.get(cacheKey) 1173 if entry is not None: 1174 if method in ("GET","HEAD"): 1175 if entry.has_key("etag"): 1176 headers.addRawHeader("if-none-match",entry["etag"]) 1177 if entry.has_key("last-modified"): 1178 headers.addRawHeader("if-modified-since",entry["last-modified"]) 1179 if method in ("PUT",): 1180 if entry.has_key("etag"): 1181 headers.addRawHeader("if-match",entry["etag"]) 1182 deferred = self._agent.request(method,uri,headers,bodyProducer) 1183 return deferred.addCallback(self._handleResponse,method=method,cacheKey=cacheKey) 1184 1185 1186 def _handleResponse(self,response,method,cacheKey): 1187 """ 1188 Check if the server response with a cache hit and read or write to the cache if nessecary. 1189 """ 1190 cache = self._cache.get(cacheKey,{}) 1191 if cache: 1192 self._cache.delete(cacheKey) 1193 if response.headers.hasHeader("etag"): 1194 cache["etag"] = response.headers.getRawHeaders("etag")[0] 1195 if response.headers.hasHeader("last-modified"): 1196 cache["last-modified"] = response.headers.getRawHeaders("last-modified")[0] 1197 self._cache.put(cacheKey,cache) 1198 1199 if response.code == 304 and method == "GET": 1200 response.code = 200 1201 response = CacheBodyProducer(response,cache) 1202 elif cache: 1203 response = CacheBodyUpdater(response,cache=self._cache,cacheKey=cacheKey) 1204 return response 1205 1006 1206 __all__ = [ 1007 1207 'PartialDownloadError', 'HTTPPageGetter', 'HTTPPageDownloader', 1008 1208 'HTTPClientFactory', 'HTTPDownloader', 'getPage', 'downloadPage', 1009 1209 'ResponseDone', 'Response', 'ResponseFailed', 'Agent', 'CookieAgent', 1010 'ContentDecoderAgent', 'GzipDecoder' ]1210 'ContentDecoderAgent', 'GzipDecoder', "CachingAgent", "MemoryCache"] -
web/iweb.py
519 519 520 520 UNKNOWN_LENGTH = u"twisted.web.iweb.UNKNOWN_LENGTH" 521 521 522 class IHTTPCache(Interface): 523 """ 524 An object representing a cache to store and satisfy http content requests. 525 To accomplish that, it stores cache entries which are in themselves C{dict} 526 objects containing the keys and values as produced by the L{Response} plus 527 a special 'content' key holding the message body, if any. 528 """ 529 530 def put(self, key, entry): 531 """ 532 Place a cache entry into the cache. 533 """ 534 535 536 def get(self, key, default=None): 537 """ 538 Reteive an entry from the cache referenced by L{key}. 539 If no such entry exists, return L{default}. 540 """ 541 542 def delete(self, key): 543 """ 544 Delete an entry L{key} from the cache. 545 """ 546 522 547 __all__ = [ 523 548 "IUsernameDigestHash", "ICredentialFactory", "IRequest", 524 "IBodyProducer", "IRenderable", "IResponse", 549 "IBodyProducer", "IRenderable", "IResponse", "IHTTPCache," 525 550 526 551 "UNKNOWN_LENGTH"]