Examples with Response org.jsoup.Connection.Response used on opensource projects

Example 6 with Response

use of org.jsoup.Connection.Response in project ripme by RipMeApp.

the class SankakuComplexRipper method getFirstPage.

@Override
public Document getFirstPage() throws IOException {
    if (albumDoc == null) {
        Response resp = Http.url(url).response();
        cookies.putAll(resp.cookies());
        albumDoc = resp.parse();
    }
    return albumDoc;
}

Example 7 with Response

use of org.jsoup.Connection.Response in project crawler-jsoup-maven by bluetata.

the class ITEYELoginApater method login.

/**
 * 模拟登陆Iteye
 * @param userName
 * @param pwd
 * @throws Exception
 */
public static void login(String userName, String pwd) throws Exception {
    // 第一次请求
    // 获取连接
    Connection con = Jsoup.connect("http://www.iteye.com/login");
    // 配置模拟浏览器
    con.header("User-Agent", "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:29.0) Gecko/20100101 Firefox/29.0");
    // 获取响应
    Response rs = con.execute();
    // 转换为Dom树
    Document d1 = Jsoup.parse(rs.body());
    // 获取form表单，可以通过查看页面源码代码得知
    List<Element> et = d1.select("#login_form");
    // 获取，cooking和表单属性，下面map存放post时的数据
    Map<String, String> datas = new HashMap<>();
    for (Element e : et.get(0).getAllElements()) {
        if (e.attr("name").equals("name")) {
            // 设置用户名
            e.attr("value", userName);
        }
        if (e.attr("name").equals("password")) {
            // 设置用户密码
            e.attr("value", pwd);
        }
        if (e.attr("name").length() > 0) {
            // 排除空值表单属性
            datas.put(e.attr("name"), e.attr("value"));
        }
    }
    /**
     * 第二次请求，post表单数据，以及cookie信息
     */
    Connection con2 = Jsoup.connect("http://www.iteye.com/login");
    con2.header("User-Agent", "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:29.0) Gecko/20100101 Firefox/29.0");
    // 设置cookie和post上面的map数据
    Response login = con2.ignoreContentType(true).method(Method.POST).data(datas).cookies(rs.cookies()).execute();
    // 打印，登陆成功后的信息
    System.out.println(login.body());
    // 登陆成功后的cookie信息，可以保存到本地，以后登陆时，只需一次登陆即可
    Map<String, String> map = login.cookies();
    for (String s : map.keySet()) {
        System.out.println(s + " : " + map.get(s));
    }
}

Example 8 with Response

use of org.jsoup.Connection.Response in project crawler-jsoup-maven by bluetata.

the class AmazonLoginApater method main.

public static void main(String[] args) {
    // grab login form page first
    try {
        // lets make data map containing all the parameters and its values found in the form
        Map<String, String> mapParamsData = new HashMap<String, String>();
        mapParamsData.put("email", "dietime1943@hotmail.com");
        mapParamsData.put("password", "152300");
        Response loginResponse = Jsoup.connect("https://passport.jd.com/new/login.aspx").userAgent(USER_AGENT).timeout(TIMEOUT_UNIT * TIMEOUT_TIMES).data(mapParamsData).method(Method.POST).followRedirects(true).execute();
        System.out.println("Fetched login page");
        // System.out.println(loginResponse.toString());
        // get the cookies from the response, which we will post to the action URL
        Map<String, String> mapLoginPageCookies = loginResponse.cookies();
        System.out.println(mapLoginPageCookies);
    } catch (Exception e) {
        e.printStackTrace();
    }
}

Example 9 with Response

use of org.jsoup.Connection.Response in project crawler-jsoup-maven by bluetata.

the class TestApiOfConnect method main.

public static void main(String[] args) throws IOException {
    // TODO Auto-generated method stub
    // Connection connection = Jsoup.connect("http://bluetata.com");
    // // connection.data("aaa","ccc");  // 这是重点
    // 
    // connection.header("Content-Type", "application/json; charset=UTF-8");  // 这是重点
    // 
    // connection.header("Accept", "text/plain, */*; q=0.01");
    // 
    // connection.timeout(15000);
    // 
    // //String body = "{\"CategoryType\":\"SiteHome\",\"ParentCategoryId\":0,\"CategoryId\":808,\"PageIndex\":2,\"TotalPostCount\":4000,\"ItemListActionName\":\"PostList\"}";
    // 
    // //connection.requestBody(body);
    // 
    // Document document = connection.post();
    String jsonBody = "{\"name\":\"ACTIVATE\",\"value\":\"E0010\"}";
    Connection connection = Jsoup.connect("http://bluetata.com/").userAgent(// User-Agent of Chrome 55
    "Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36").referrer("http://bluetata.com/").header("Content-Type", "application/json; charset=UTF-8").header("Accept", "text/plain, */*; q=0.01").header("Accept-Encoding", "gzip,deflate,sdch").header("Accept-Language", "es-ES,es;q=0.8").header("Connection", "keep-alive").header("X-Requested-With", "XMLHttpRequest").requestBody(jsonBody).maxBodySize(100).timeout(1000 * 10).method(Connection.Method.POST);
    Response response = connection.execute();
}

Example 10 with Response

use of org.jsoup.Connection.Response in project portfolio by buchen.

the class YahooFinanceQuoteFeed method loadCrump.

private Crumb loadCrump(String tickerSymbol) throws IOException {
    // $NON-NLS-1$
    String url = MessageFormat.format("https://de.finance.yahoo.com/quote/{0}/history?p={0}", tickerSymbol);
    Response response = // 
    Jsoup.connect(url).userAgent(OnlineHelper.getUserAgent()).header("Accept", // $NON-NLS-1$ //$NON-NLS-2$
    "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8").header("Accept-Language", // //$NON-NLS-1$ //$NON-NLS-2$
    "en-US,en;q=0.5").timeout(// 
    30000).execute();
    // $NON-NLS-1$
    String KEY = "\"CrumbStore\":{\"crumb\":\"";
    String body = response.body();
    int startIndex = body.indexOf(KEY);
    if (startIndex < 0)
        throw new IOException(Messages.MsgErrorNoCrumbFound);
    int endIndex = body.indexOf('"', startIndex + KEY.length());
    if (endIndex < 0)
        throw new IOException(Messages.MsgErrorNoCrumbFound);
    String crumb = body.substring(startIndex + KEY.length(), endIndex);
    crumb = StringEscapeUtils.unescapeJava(crumb);
    return new Crumb(crumb, response.cookies());
}