Search in sources :

Example 6 with Response

use of com.cv4j.netdiscovery.core.domain.Response in project NetDiscovery by fengzhizi715.

the class VertxDownloader method download.

public Maybe<Response> download(Request request) {
    WebClientOptions options = initWebClientOptions(request);
    webClient = WebClient.create(vertx, options);
    HttpRequest<Buffer> httpRequest = null;
    if (request.getHttpMethod() == HttpMethod.GET) {
        if ("http".equals(request.getUrlParser().getProtocol())) {
            httpRequest = webClient.getAbs(request.getUrl());
        } else if ("https".equals(request.getUrlParser().getProtocol())) {
            httpRequest = webClient.get(443, request.getUrlParser().getHost(), Preconditions.isNotBlank(request.getUrlParser().getQuery()) ? request.getUrlParser().getPath() + "?" + request.getUrlParser().getQuery() : request.getUrlParser().getPath()).ssl(true);
        }
    } else if (request.getHttpMethod() == HttpMethod.POST) {
        if ("http".equals(request.getUrlParser().getProtocol())) {
            httpRequest = webClient.postAbs(request.getUrl());
        } else if ("https".equals(request.getUrlParser().getProtocol())) {
            httpRequest = webClient.post(443, request.getUrlParser().getHost(), Preconditions.isNotBlank(request.getUrlParser().getQuery()) ? request.getUrlParser().getPath() + "?" + request.getUrlParser().getQuery() : request.getUrlParser().getPath()).ssl(true);
        }
    }
    // 设置请求头header
    if (Preconditions.isNotBlank(header)) {
        for (Map.Entry<String, String> entry : header.entrySet()) {
            httpRequest.putHeader(entry.getKey(), entry.getValue());
        }
    }
    // 针对post请求,需要对header添加一些信息
    if (request.getHttpMethod() == HttpMethod.POST) {
        if (Preconditions.isNotBlank(request.getHttpRequestBody()) && Preconditions.isNotBlank(request.getHttpRequestBody().getContentType())) {
            httpRequest.putHeader("Content-type", request.getHttpRequestBody().getContentType());
        }
    }
    String charset = null;
    if (Preconditions.isNotBlank(request.getCharset())) {
        charset = request.getCharset();
    } else {
        charset = Constant.UTF_8;
    }
    HttpRequest<String> stringHttpRequest = httpRequest.as(BodyCodec.string(charset));
    Single<HttpResponse<String>> httpResponseSingle = null;
    if (request.getHttpMethod() == HttpMethod.GET) {
        httpResponseSingle = stringHttpRequest.rxSend();
    } else if (request.getHttpMethod() == HttpMethod.POST) {
        if (Preconditions.isNotBlank(request.getHttpRequestBody())) {
            Buffer buffer = Buffer.buffer();
            buffer.getDelegate().appendBytes(request.getHttpRequestBody().getBody());
            httpResponseSingle = stringHttpRequest.rxSendBuffer(buffer);
        } else {
            httpResponseSingle = stringHttpRequest.rxSend();
        }
    }
    return httpResponseSingle.toMaybe().map(new Function<HttpResponse<String>, Response>() {

        @Override
        public Response apply(HttpResponse<String> stringHttpResponse) throws Exception {
            String html = stringHttpResponse.body();
            Response response = new Response();
            response.setContent(html.getBytes());
            response.setStatusCode(stringHttpResponse.statusCode());
            response.setContentType(stringHttpResponse.getHeader("Content-Type"));
            if (request.isSaveCookie()) {
                // save cookies
                if (Preconditions.isNotBlank(stringHttpResponse.cookies())) {
                    CookieGroup cookieGroup = CookieManager.getInsatance().getCookieGroup(request.getUrlParser().getHost());
                    if (cookieGroup == null) {
                        cookieGroup = new CookieGroup(request.getUrlParser().getHost());
                        for (String cookieStr : stringHttpResponse.cookies()) {
                            String[] segs = cookieStr.split(";");
                            if (Preconditions.isNotBlank(segs)) {
                                for (String seg : segs) {
                                    String[] pairs = seg.trim().split("\\=");
                                    if (pairs.length == 2) {
                                        cookieSet.add(new Cookie(pairs[0], pairs[1]));
                                    }
                                }
                            }
                        }
                        cookieGroup.putAllCookies(cookieSet);
                        CookieManager.getInsatance().addCookieGroup(cookieGroup);
                    } else {
                        for (String cookieStr : stringHttpResponse.cookies()) {
                            String[] segs = cookieStr.split(";");
                            if (Preconditions.isNotBlank(segs)) {
                                for (String seg : segs) {
                                    String[] pairs = seg.trim().split("\\=");
                                    if (pairs.length == 2) {
                                        cookieSet.add(new Cookie(pairs[0], pairs[1]));
                                    }
                                }
                            }
                        }
                        cookieGroup.putAllCookies(cookieSet);
                    }
                }
            }
            return response;
        }
    });
}
Also used : Buffer(io.vertx.reactivex.core.buffer.Buffer) Cookie(com.cv4j.netdiscovery.core.cookies.Cookie) WebClientOptions(io.vertx.ext.web.client.WebClientOptions) HttpResponse(io.vertx.reactivex.ext.web.client.HttpResponse) HttpResponse(io.vertx.reactivex.ext.web.client.HttpResponse) Response(com.cv4j.netdiscovery.core.domain.Response) CookieGroup(com.cv4j.netdiscovery.core.cookies.CookieGroup) Map(java.util.Map)

Aggregations

Response (com.cv4j.netdiscovery.core.domain.Response)6 MaybeEmitter (io.reactivex.MaybeEmitter)4 MaybeOnSubscribe (io.reactivex.MaybeOnSubscribe)4 IOException (java.io.IOException)3 Map (java.util.Map)3 Cookie (com.cv4j.netdiscovery.core.cookies.Cookie)2 CookieGroup (com.cv4j.netdiscovery.core.cookies.CookieGroup)2 Request (com.cv4j.netdiscovery.core.domain.Request)2 InputStream (java.io.InputStream)2 MalformedURLException (java.net.MalformedURLException)2 URL (java.net.URL)2 Constant (com.cv4j.netdiscovery.core.config.Constant)1 HttpRequestBody (com.cv4j.netdiscovery.core.domain.HttpRequestBody)1 Page (com.cv4j.netdiscovery.core.domain.Page)1 Downloader (com.cv4j.netdiscovery.core.downloader.Downloader)1 VertxDownloader (com.cv4j.netdiscovery.core.downloader.vertx.VertxDownloader)1 Parser (com.cv4j.netdiscovery.core.parser.Parser)1 Html (com.cv4j.netdiscovery.core.parser.selector.Html)1 Json (com.cv4j.netdiscovery.core.parser.selector.Json)1 Pipeline (com.cv4j.netdiscovery.core.pipeline.Pipeline)1