Using cookies with twisted.web.client

Question

I am trying to make a web client application using a twisted but with some cookie problems. Does anyone have an example that I can look at?

Answer 1

While it’s true that it getPagedoesn’t just allow direct access to the request or response headers (just one example of how getPagenot a state-of-the-art API), cookies are actually supported.

cookies = {cookies: tosend}
d = getPage(url, cookies=cookies)
def cbPage(result):
    print 'Look at my cookies:', cookies
d.addCallback(cbPage)

Any cookies in the dictionary, when they are sent to getPage, will be sent. Any new cookies that the server sets in response to the request will be added to the dictionary.

, getPage, getPage cookies ! , **kwargs, cookies: , getPage, , HTTPClientFactory.__init__. , , getPage.

Answer 2

, afaict twisted.web.client.HTTPClientFactory, twisted.web.client.getPage(), , -. :

from twisted.web import client

def getPage(url, contextFactory=None, *args, **kwargs):
    fact = client._makeGetterFactory(
        url,
        HTTPClientFactory,
        contextFactory=contextFactory,
        *args, **kwargs)
    return fact.deferred.addCallback(lambda data: (data, fact.response_headers))

Answer 3

from twisted.internet import reactor
from twisted.web import client

def getPage(url, contextFactory=None, *args, **kwargs):
    return client._makeGetterFactory(
        url,
        CustomHTTPClientFactory,
        contextFactory=contextFactory,
        *args, **kwargs).deferred

class CustomHTTPClientFactory(client.HTTPClientFactory):

    def __init__(self,url, method='GET', postdata=None, headers=None,
                 agent="Twisted PageGetter", timeout=0, cookies=None,
                 followRedirect=1, redirectLimit=20):
        client.HTTPClientFactory.__init__(self, url, method, postdata,
                                          headers, agent, timeout, cookies,
                                          followRedirect, redirectLimit)

    def page(self, page):
        if self.waiting:
            self.waiting = 0
            res = {}
            res['page'] = page
            res['headers'] = self.response_headers
            res['cookies'] = self.cookies
            self.deferred.callback(res)

if __name__ == '__main__':
    def cback(result):
        for k in result:
            print k, '==>', result[k]
        reactor.stop()

    def eback(error):
        print error.getTraceback()
        reactor.stop()

    d = getPage('http://example.com', agent='example web client', 
                 cookies={ 'some' : 'cookie' } )
    d.addCallback(cback)
    d.addErrback(eback)

    reactor.run()