Getting an AJAX Form Using Python Queries

Question

I am trying to go through the form page http://dq.ndc.bsnl.co.in/bsnl-web/residentialSearch.seam using the python query module.

The problem I assume is AJAX in the form field. And I really have no idea how to send a request with Python requests for this. I know that this can be done through Selenium, but I need to do this using queries.

Here is my current code:

import requests
headers = {'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:27.0) Gecko/20100101 Firefox/27.0'
           }
payload = {
    "residential": "residential",
    "residential:j_id12": "",
    "residential:firstField": 'a',
    "residential:criteria1": "3",
    "residential:city": "ASIND",
    "residential:button1": "residential:button1",
    "residential:suggestionBoxId_selection": "",
    "javax.faces.ViewState": "j_id1"

}
with requests.Session() as s:
    # print s.headers
    print s.get('http://dq.ndc.bsnl.co.in/bsnl-web/residentialSearch.seam')
    print s.headers
    print s.cookies
    resp = s.post(
        'http://dq.ndc.bsnl.co.in/bsnl-web/residentialSearch.seam',
        data=payload, headers=headers)

    print resp.text

Answer 1

. AJAXREQUEST , , . . : , . Page x of y.

import re
import requests
import requests.models

# non-standard conform redirect:
requests.Response.is_redirect = property(lambda self: (
    'location' in self.headers and (
        self.status_code in requests.models.REDIRECT_STATI or
        self.headers.get('Ajax-Response', '') == 'redirect'
)))

headers = {
    'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:27.0) Gecko/20100101 Firefox/27.0'
}
payload = {
    "AJAXREQUEST": "loader2",
    "residential": "residential",
    "residential:j_id12": "",
    "residential:firstField": 'a',
    "residential:criteria1": "3",
    "residential:city": "ASIND",
    "residential:button1": "residential:button1",
    "residential:suggestionBoxId_selection": "",
    "javax.faces.ViewState": "j_id1"

}

with requests.Session() as s:
    print s.get('http://dq.ndc.bsnl.co.in/bsnl-web/residentialSearch.seam')
    print s.headers
    print s.cookies
    resp = s.post(
        'http://dq.ndc.bsnl.co.in/bsnl-web/residentialSearch.seam',
        data=payload, headers=headers)

    while True:
        # do data processing
        for l in resp.text.split("subscriber');")[1:]: print l[2:].split('<')[0]

        # look for next page
        current, last = re.search('Page (\d+) of (\d+)', resp.text).groups()
        if int(current) == int(last):
            break

        resp = s.post('http://dq.ndc.bsnl.co.in/bsnl-web/resSrchDtls.seam',
            data={'AJAXREQUEST':'_viewRoot',
                'j_id10':'j_id10',
                'javax.faces.ViewState':'j_id2',
                'j_id10:PGDOWNLink':'j_id10:PGDOWNLink',
            }, headers=headers)