HTTPclient POST with problematic website

Question

I am trying to extract some data from a website .

I wrote a java class that seems to work very well with many sites, but it doesn’t work with this particular site that uses large javascript in the input fomr.

As you can see from the code, I have specified input fields that take the name from the HTML source, but maybe this site does not accept this kind of POST request?

How can I simulate user interaction to get generated HTML?

package com.transport.urlRetriver;

import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.ArrayList;

import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.NameValuePair;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.message.BasicNameValuePair;

public class UrlRetriver {


    String stationPoller (String url, ArrayList<NameValuePair> params) {

        HttpPost postRequest;
        HttpResponse response;
        HttpEntity entity;
        String result = null;

        DefaultHttpClient httpClient = new DefaultHttpClient();


        try {

            postRequest = new HttpPost(url);

            postRequest.setEntity((HttpEntity) new UrlEncodedFormEntity(params));
            response = httpClient.execute(postRequest);

            entity = response.getEntity();

            if(entity != null){
              InputStream inputStream = entity.getContent();
              result = convertStreamToString(inputStream);
            }



        } catch (Exception e) {

            result = "We had a problem";

        } finally {

            httpClient.getConnectionManager().shutdown();

        }



        return result;

    }





    void ATMtravelPoller () {




        ArrayList<NameValuePair> params = new ArrayList<NameValuePair>(2);

        String url = "http://www.atm-mi.it/it/Pagine/default.aspx";

        params.add(new BasicNameValuePair("ctl00$SPWebPartManager1$g_afa5adbb_5b60_4e50_8da2_212a1d36e49c$txt_address_s", "Viale romagna 1"));

        params.add(new BasicNameValuePair("ctl00$SPWebPartManager1$g_afa5adbb_5b60_4e50_8da2_212a1d36e49c$txt_address_e", "Viale Toscana 20"));

        params.add(new BasicNameValuePair("sf_method", "POST"));

        String result = stationPoller(url, params);

        saveToFile(result, "/home/rachele/Documents/atm/out4.html");

    }

    static void saveToFile(String toFile, String pos){
          try{
                // Create file 
                FileWriter fstream = new FileWriter(pos);
                BufferedWriter out = new BufferedWriter(fstream);
                out.write(toFile);
                //Close the output stream
                out.close();
                }catch (Exception e){//Catch exception if any
                  System.err.println("Error: " + e.getMessage());
                }
              }

    private static String convertStreamToString(InputStream is) {
          BufferedReader reader = new BufferedReader(new InputStreamReader(is));
          StringBuilder stringBuilder = new StringBuilder();

          String line = null;
          try {
            while ((line = reader.readLine()) != null) {
              stringBuilder.append(line + "\n");
            }
          } catch (IOException e) {
            e.printStackTrace();
          } finally {
            try {
              is.close();
            } catch (IOException e) {
              e.printStackTrace();
            }
          }
          return stringBuilder.toString();
        }

}

Answer 1

, javascript , . , .