Examples with ProxyPool com.cdeledu.crawler.SocialNetwork.proxy.entity.ProxyPool used on opensource projects

Example 1 with ProxyPool

use of com.cdeledu.crawler.SocialNetwork.proxy.entity.ProxyPool in project wechat by dllwh.

the class Ip181Helper method getDetailInfo.

/**
 * @方法描述: 获取详细信息
 * @param data
 * @return
 */
private static ProxyPool getDetailInfo(Element data) {
    ProxyPool proxyIP = new ProxyPool();
    Elements tdData = data.getElementsByTag("td");
    proxyIP.setIp(tdData.get(0).text());
    if (StringUtils.isNoneBlank(tdData.get(1).text())) {
        proxyIP.setPort(Integer.valueOf(tdData.get(1).text()));
    }
    proxyIP.setProtocolType(tdData.get(3).text().toLowerCase().split(","));
    proxyIP.setPosition(tdData.get(5).text());
    return proxyIP;
}

Example 2 with ProxyPool

use of com.cdeledu.crawler.SocialNetwork.proxy.entity.ProxyPool in project wechat by dllwh.

the class XdailiProxyHelper method getFreeIps.

/**
 * @方法描述: 获取免费代理IP
 * @param page
 * @return
 * @throws Exception
 */
public static String getFreeIps(int page) throws Exception {
    if (page < 1) {
        page = 1;
    }
    getFreeIpListUrl = String.format(getFreeIpListUrl, page);
    String result = HttpURLConnHelper.getInstance().sendGetRequest(getFreeIpListUrl);
    JSONObject _result = JSONObject.fromObject(result);
    List<Map<String, Object>> resultList = Lists.newArrayList();
    if (_result.has("ERRORCODE")) {
        if ("0".equals(_result.getString("ERRORCODE"))) {
            String childResult = _result.getString("RESULT");
            if (StringUtils.isNoneBlank(childResult)) {
                JSONObject _cResult = JSONObject.fromObject(childResult);
                JSONArray _Jarray = JSONArray.fromObject(_cResult.getString("rows"));
                JSONObject job = null;
                ProxyPool proxyIP = null;
                for (int i = 0; i < _Jarray.size(); i++) {
                    try {
                        job = _Jarray.getJSONObject(i);
                        proxyIP = new ProxyPool();
                        proxyIP.setIp(job.getString("ip"));
                        proxyIP.setPort(Integer.valueOf(job.getString("port")));
                        String[] position = job.getString("position").split(" ");
                        proxyIP.setPosition(position[0]);
                        proxyIP.setIsp(position[1]);
                        proxyIP.setProtocolType(job.getString("type").split("/"));
                        resultList.add(MapUtilHelper.beanToMap(proxyIP));
                    } catch (Exception e) {
                        logger.error(e);
                    }
                    System.err.println();
                }
            }
        }
    }
    return JsonMapper.toJsonString(resultList);
}

Example 3 with ProxyPool

use of com.cdeledu.crawler.SocialNetwork.proxy.entity.ProxyPool in project wechat by dllwh.

the class XiciDailiHelper method getProxyList.

private static String getProxyList(String url) throws Exception {
    List<Map<String, Object>> resultList = null;
    resultList = Lists.newArrayList();
    Document document = Jsoup.parse(webCrawler.crawl(url, crawlPara));
    Elements dataTable = document.body().select("table#ip_list").first().select("tr");
    try {
        ProxyPool proxyIP = null;
        for (int i = 1; i < dataTable.size(); i++) {
            try {
                proxyIP = new ProxyPool();
                Elements tdData = dataTable.get(i).getElementsByTag("td");
                proxyIP.setIp(tdData.get(1).text());
                if (StringUtils.isNoneBlank(tdData.get(2).text())) {
                    proxyIP.setPort(Integer.valueOf(tdData.get(2).text()));
                }
                proxyIP.setPosition(tdData.get(3).text());
                proxyIP.setProtocolType(tdData.get(5).text().toLowerCase().split(","));
                resultList.add(MapUtilHelper.beanToMap(proxyIP));
            } catch (Exception e) {
                if (logger.isDebugEnabled()) {
                    e.printStackTrace();
                }
                if (logger.isErrorEnabled()) {
                    e.printStackTrace();
                }
            }
        }
    } catch (Exception ex) {
        ex.printStackTrace();
    }
    return JsonMapper.toJsonString(resultList);
}

Example 4 with ProxyPool

use of com.cdeledu.crawler.SocialNetwork.proxy.entity.ProxyPool in project wechat by dllwh.

the class Data5uHelper method getProxyList.

private static String getProxyList(String url) throws Exception {
    crawlPara.setReqmethod("GET");
    String result = webCrawler.crawl(BASE_URL + "index.shtml", crawlPara);
    Document document = Jsoup.parse(result);
    Elements wList = document.select("body > div.wlist >ul > li:eq(1) > ul");
    List<Map<String, Object>> resultList = Lists.newArrayList();
    ProxyPool proxyIP = null;
    for (int i = 1; i < wList.size(); i++) {
        try {
            proxyIP = new ProxyPool();
            Elements spanData = wList.get(i).select("span");
            proxyIP.setIp(spanData.get(0).text());
            proxyIP.setPort(Integer.valueOf(spanData.get(1).text()));
            proxyIP.setProtocolType(spanData.get(3).text().split(","));
            proxyIP.setCountry(spanData.get(3).text());
            proxyIP.setPosition(spanData.get(4).text() + spanData.get(5).text());
            proxyIP.setIsp(spanData.get(6).text());
            resultList.add(MapUtilHelper.beanToMap(proxyIP));
        } catch (Exception e) {
            if (logger.isDebugEnabled()) {
                e.printStackTrace();
            }
            if (logger.isErrorEnabled()) {
                e.printStackTrace();
            }
        }
    }
    return JsonMapper.toJsonString(resultList);
}