Examples with CachingGZIPInputStream lucee.commons.net.http.httpclient.CachingGZIPInputStream used on opensource projects

Example 1 with CachingGZIPInputStream

use of lucee.commons.net.http.httpclient.CachingGZIPInputStream in project Lucee by lucee.

the class HttpGetWithBody method _doEndTag.

private void _doEndTag() throws PageException, IOException {
    long start = System.nanoTime();
    HttpClientBuilder builder = HTTPEngine4Impl.getHttpClientBuilder();
    ssl(builder);
    // redirect
    if (redirect)
        builder.setRedirectStrategy(new DefaultRedirectStrategy());
    else
        builder.disableRedirectHandling();
    // cookies
    BasicCookieStore cookieStore = new BasicCookieStore();
    builder.setDefaultCookieStore(cookieStore);
    ConfigWeb cw = pageContext.getConfig();
    HttpRequestBase req = null;
    HttpContext httpContext = null;
    CacheHandler cacheHandler = null;
    String cacheId = null;
    // HttpRequestBase req = init(pageContext.getConfig(),this,client,params,url,port);
    {
        if (StringUtil.isEmpty(charset, true))
            charset = ((PageContextImpl) pageContext).getWebCharset().name();
        else
            charset = charset.trim();
        // check if has fileUploads
        boolean doUploadFile = false;
        for (int i = 0; i < this.params.size(); i++) {
            if ((this.params.get(i)).getType() == HttpParamBean.TYPE_FILE) {
                doUploadFile = true;
                break;
            }
        }
        // parse url (also query string)
        int len = this.params.size();
        StringBuilder sbQS = new StringBuilder();
        for (int i = 0; i < len; i++) {
            HttpParamBean param = this.params.get(i);
            int type = param.getType();
            // URL
            if (type == HttpParamBean.TYPE_URL) {
                if (sbQS.length() > 0)
                    sbQS.append('&');
                sbQS.append(param.getEncoded() ? urlenc(param.getName(), charset) : param.getName());
                sbQS.append('=');
                sbQS.append(param.getEncoded() ? urlenc(param.getValueAsString(), charset) : param.getValueAsString());
            }
        }
        String host = null;
        HttpHost httpHost;
        try {
            URL _url = HTTPUtil.toURL(url, port, encoded);
            httpHost = new HttpHost(_url.getHost(), _url.getPort());
            host = _url.getHost();
            url = _url.toExternalForm();
            if (sbQS.length() > 0) {
                // no existing QS
                if (StringUtil.isEmpty(_url.getQuery())) {
                    url += "?" + sbQS;
                } else {
                    url += "&" + sbQS;
                }
            }
        } catch (MalformedURLException mue) {
            throw Caster.toPageException(mue);
        }
        // cache
        if (cachedWithin != null) {
            cacheId = createCacheId();
            cacheHandler = pageContext.getConfig().getCacheHandlerCollection(Config.CACHE_TYPE_HTTP, null).getInstanceMatchingObject(cachedWithin, null);
            if (cacheHandler instanceof CacheHandlerPro) {
                CacheItem cacheItem = ((CacheHandlerPro) cacheHandler).get(pageContext, cacheId, cachedWithin);
                if (cacheItem instanceof HTTPCacheItem) {
                    pageContext.setVariable(result, ((HTTPCacheItem) cacheItem).getData());
                    return;
                }
            } else if (cacheHandler != null) {
                // TODO this else block can be removed when all cache handlers implement CacheHandlerPro
                CacheItem cacheItem = cacheHandler.get(pageContext, cacheId);
                if (cacheItem instanceof HTTPCacheItem) {
                    pageContext.setVariable(result, ((HTTPCacheItem) cacheItem).getData());
                    return;
                }
            }
        }
        // cache not found, process and cache result if needed
        // select best matching method (get,post, post multpart (file))
        boolean isBinary = false;
        boolean doMultiPart = doUploadFile || this.multiPart;
        HttpEntityEnclosingRequest eeReqPost = null;
        HttpEntityEnclosingRequest eeReq = null;
        if (this.method == METHOD_GET) {
            req = new HttpGetWithBody(url);
            eeReq = (HttpEntityEnclosingRequest) req;
        } else if (this.method == METHOD_HEAD) {
            req = new HttpHead(url);
        } else if (this.method == METHOD_DELETE) {
            isBinary = true;
            req = new HttpDeleteWithBody(url);
            eeReq = (HttpEntityEnclosingRequest) req;
        } else if (this.method == METHOD_PUT) {
            isBinary = true;
            HttpPut put = new HttpPut(url);
            eeReqPost = put;
            req = put;
            eeReq = put;
        } else if (this.method == METHOD_TRACE) {
            isBinary = true;
            req = new HttpTrace(url);
        } else if (this.method == METHOD_OPTIONS) {
            isBinary = true;
            req = new HttpOptions(url);
        } else if (this.method == METHOD_PATCH) {
            isBinary = true;
            eeReq = HTTPPatchFactory.getHTTPPatch(url);
            req = (HttpRequestBase) eeReq;
        } else {
            isBinary = true;
            eeReqPost = new HttpPost(url);
            req = (HttpPost) eeReqPost;
            eeReq = eeReqPost;
        }
        boolean hasForm = false;
        boolean hasBody = false;
        boolean hasContentType = false;
        // Set http params
        ArrayList<FormBodyPart> parts = new ArrayList<FormBodyPart>();
        StringBuilder acceptEncoding = new StringBuilder();
        java.util.List<NameValuePair> postParam = eeReqPost != null ? new ArrayList<NameValuePair>() : null;
        for (int i = 0; i < len; i++) {
            HttpParamBean param = this.params.get(i);
            int type = param.getType();
            // URL
            if (type == HttpParamBean.TYPE_URL) {
            // listQS.add(new BasicNameValuePair(translateEncoding(param.getName(), http.charset),translateEncoding(param.getValueAsString(),
            // http.charset)));
            } else // Form
            if (type == HttpParamBean.TYPE_FORM) {
                hasForm = true;
                if (this.method == METHOD_GET)
                    throw new ApplicationException("httpparam with type formfield can only be used when the method attribute of the parent http tag is set to post");
                if (eeReqPost != null) {
                    if (doMultiPart) {
                        parts.add(new FormBodyPart(param.getName(), new StringBody(param.getValueAsString(), CharsetUtil.toCharset(charset))));
                    } else {
                        postParam.add(new BasicNameValuePair(param.getName(), param.getValueAsString()));
                    }
                }
            // else if(multi!=null)multi.addParameter(param.getName(),param.getValueAsString());
            } else // CGI
            if (type == HttpParamBean.TYPE_CGI) {
                if (param.getEncoded())
                    req.addHeader(urlenc(param.getName(), charset), urlenc(param.getValueAsString(), charset));
                else
                    req.addHeader(param.getName(), param.getValueAsString());
            } else // Header
            if (type == HttpParamBean.TYPE_HEADER) {
                if (param.getName().equalsIgnoreCase("content-type"))
                    hasContentType = true;
                if (param.getName().equalsIgnoreCase("Content-Length")) {
                } else if (param.getName().equalsIgnoreCase("Accept-Encoding")) {
                    acceptEncoding.append(headerValue(param.getValueAsString()));
                    acceptEncoding.append(", ");
                } else
                    req.addHeader(param.getName(), headerValue(param.getValueAsString()));
            } else // Cookie
            if (type == HttpParamBean.TYPE_COOKIE) {
                HTTPEngine4Impl.addCookie(cookieStore, host, param.getName(), param.getValueAsString(), "/", charset);
            } else // File
            if (type == HttpParamBean.TYPE_FILE) {
                hasForm = true;
                if (this.method == METHOD_GET)
                    throw new ApplicationException("httpparam type file can't only be used, when method of the tag http equal post");
                // if(param.getFile()==null) throw new ApplicationException("httpparam type file can't only be used, when method of the tag http equal
                // post");
                String strCT = getContentType(param);
                ContentType ct = HTTPUtil.toContentType(strCT, null);
                String mt = "text/xml";
                if (ct != null && !StringUtil.isEmpty(ct.getMimeType(), true))
                    mt = ct.getMimeType();
                String cs = charset;
                if (ct != null && !StringUtil.isEmpty(ct.getCharset(), true))
                    cs = ct.getCharset();
                if (doMultiPart) {
                    try {
                        Resource res = param.getFile();
                        parts.add(new FormBodyPart(param.getName(), new ResourceBody(res, mt, res.getName(), cs)));
                    // parts.add(new ResourcePart(param.getName(),new ResourcePartSource(param.getFile()),getContentType(param),_charset));
                    } catch (FileNotFoundException e) {
                        throw new ApplicationException("can't upload file, path is invalid", e.getMessage());
                    }
                }
            } else // XML
            if (type == HttpParamBean.TYPE_XML) {
                ContentType ct = HTTPUtil.toContentType(param.getMimeType(), null);
                String mt = "text/xml";
                if (ct != null && !StringUtil.isEmpty(ct.getMimeType(), true))
                    mt = ct.getMimeType();
                String cs = charset;
                if (ct != null && !StringUtil.isEmpty(ct.getCharset(), true))
                    cs = ct.getCharset();
                hasBody = true;
                hasContentType = true;
                req.addHeader("Content-type", mt + "; charset=" + cs);
                if (eeReq == null)
                    throw new ApplicationException("type xml is only supported for methods get, delete, post, and put");
                HTTPEngine4Impl.setBody(eeReq, param.getValueAsString(), mt, cs);
            } else // Body
            if (type == HttpParamBean.TYPE_BODY) {
                ContentType ct = HTTPUtil.toContentType(param.getMimeType(), null);
                String mt = null;
                if (ct != null && !StringUtil.isEmpty(ct.getMimeType(), true))
                    mt = ct.getMimeType();
                String cs = charset;
                if (ct != null && !StringUtil.isEmpty(ct.getCharset(), true))
                    cs = ct.getCharset();
                hasBody = true;
                if (eeReq == null)
                    throw new ApplicationException("type body is only supported for methods get, delete, post, and put");
                HTTPEngine4Impl.setBody(eeReq, param.getValue(), mt, cs);
            } else {
                throw new ApplicationException("invalid type [" + type + "]");
            }
        }
        // post params
        if (postParam != null && postParam.size() > 0)
            eeReqPost.setEntity(new org.apache.http.client.entity.UrlEncodedFormEntity(postParam, charset));
        if (compression) {
            acceptEncoding.append("gzip");
        } else {
            acceptEncoding.append("deflate;q=0");
            req.setHeader("TE", "deflate;q=0");
        }
        req.setHeader("Accept-Encoding", acceptEncoding.toString());
        // multipart
        if (doMultiPart && eeReq != null) {
            hasContentType = true;
            boolean doIt = true;
            if (!this.multiPart && parts.size() == 1) {
                ContentBody body = parts.get(0).getBody();
                if (body instanceof StringBody) {
                    StringBody sb = (StringBody) body;
                    try {
                        org.apache.http.entity.ContentType ct = org.apache.http.entity.ContentType.create(sb.getMimeType(), sb.getCharset());
                        String str = IOUtil.toString(sb.getReader());
                        StringEntity entity = new StringEntity(str, ct);
                        eeReq.setEntity(entity);
                    } catch (IOException e) {
                        throw Caster.toPageException(e);
                    }
                    doIt = false;
                }
            }
            if (doIt) {
                MultipartEntityBuilder mpeBuilder = MultipartEntityBuilder.create().setStrictMode();
                // enabling the line below will append charset=... to the Content-Type header
                // if (!StringUtil.isEmpty(charset, true))
                // mpeBuilder.setCharset(CharsetUtil.toCharset(charset));
                Iterator<FormBodyPart> it = parts.iterator();
                while (it.hasNext()) {
                    FormBodyPart part = it.next();
                    mpeBuilder.addPart(part);
                }
                eeReq.setEntity(mpeBuilder.build());
            }
        // eem.setRequestEntity(new MultipartRequestEntityFlex(parts.toArray(new Part[parts.size()]), eem.getParams(),http.multiPartType));
        }
        if (hasBody && hasForm)
            throw new ApplicationException("mixing httpparam  type file/formfield and body/XML is not allowed");
        if (!hasContentType) {
            if (isBinary) {
                if (hasBody)
                    req.addHeader("Content-type", "application/octet-stream");
                else
                    req.addHeader("Content-type", "application/x-www-form-urlencoded; charset=" + charset);
            } else {
                if (hasBody)
                    req.addHeader("Content-type", "text/html; charset=" + charset);
            }
        }
        // set User Agent
        if (!hasHeaderIgnoreCase(req, "User-Agent"))
            req.setHeader("User-Agent", this.useragent);
        // set timeout
        setTimeout(builder, checkRemainingTimeout());
        // set Username and Password
        if (this.username != null) {
            if (this.password == null)
                this.password = "";
            if (AUTH_TYPE_NTLM == this.authType) {
                if (StringUtil.isEmpty(this.workStation, true))
                    throw new ApplicationException("attribute workstation is required when authentication type is [NTLM]");
                if (StringUtil.isEmpty(this.domain, true))
                    throw new ApplicationException("attribute domain is required when authentication type is [NTLM]");
                HTTPEngine4Impl.setNTCredentials(builder, this.username, this.password, this.workStation, this.domain);
            } else
                httpContext = HTTPEngine4Impl.setCredentials(builder, httpHost, this.username, this.password, preauth);
        }
        // set Proxy
        ProxyData proxy = null;
        if (!StringUtil.isEmpty(this.proxyserver)) {
            proxy = ProxyDataImpl.getInstance(this.proxyserver, this.proxyport, this.proxyuser, this.proxypassword);
        }
        if (pageContext.getConfig().isProxyEnableFor(host)) {
            proxy = pageContext.getConfig().getProxyData();
        }
        HTTPEngine4Impl.setProxy(builder, req, proxy);
    }
    CloseableHttpClient client = null;
    try {
        if (httpContext == null)
            httpContext = new BasicHttpContext();
        Struct cfhttp = new StructImpl();
        cfhttp.setEL(ERROR_DETAIL, "");
        pageContext.setVariable(result, cfhttp);
        // ///////////////////////////////////////// EXECUTE /////////////////////////////////////////////////
        client = builder.build();
        Executor4 e = new Executor4(pageContext, this, client, httpContext, req, redirect);
        HTTPResponse4Impl rsp = null;
        if (timeout == null || timeout.getMillis() <= 0) {
            try {
                rsp = e.execute(httpContext);
            } catch (Throwable t) {
                ExceptionUtil.rethrowIfNecessary(t);
                if (!throwonerror) {
                    if (t instanceof SocketTimeoutException)
                        setRequestTimeout(cfhttp);
                    else
                        setUnknownHost(cfhttp, t);
                    return;
                }
                throw toPageException(t, rsp);
            }
        } else {
            e.start();
            try {
                synchronized (this) {
                    // print.err(timeout);
                    this.wait(timeout.getMillis());
                }
            } catch (InterruptedException ie) {
                throw Caster.toPageException(ie);
            }
            if (e.t != null) {
                if (!throwonerror) {
                    setUnknownHost(cfhttp, e.t);
                    return;
                }
                throw toPageException(e.t, rsp);
            }
            rsp = e.response;
            if (!e.done) {
                req.abort();
                if (throwonerror)
                    throw new HTTPException("408 Request Time-out", "a timeout occurred in tag http", 408, "Time-out", rsp == null ? null : rsp.getURL());
                setRequestTimeout(cfhttp);
                return;
            // throw new ApplicationException("timeout");
            }
        }
        // ///////////////////////////////////////// EXECUTE /////////////////////////////////////////////////
        Charset responseCharset = CharsetUtil.toCharset(rsp.getCharset());
        int statCode = 0;
        // Write Response Scope
        // String rawHeader=httpMethod.getStatusLine().toString();
        String mimetype = null;
        String contentEncoding = null;
        // status code
        cfhttp.set(STATUSCODE, ((rsp.getStatusCode() + " " + rsp.getStatusText()).trim()));
        cfhttp.set(STATUS_CODE, new Double(statCode = rsp.getStatusCode()));
        cfhttp.set(STATUS_TEXT, (rsp.getStatusText()));
        cfhttp.set(HTTP_VERSION, (rsp.getProtocolVersion()));
        // responseHeader
        lucee.commons.net.http.Header[] headers = rsp.getAllHeaders();
        StringBuffer raw = new StringBuffer(rsp.getStatusLine() + " ");
        Struct responseHeader = new StructImpl();
        Struct cookie;
        Array setCookie = new ArrayImpl();
        Query cookies = new QueryImpl(new String[] { "name", "value", "path", "domain", "expires", "secure", "httpOnly" }, 0, "cookies");
        for (int i = 0; i < headers.length; i++) {
            lucee.commons.net.http.Header header = headers[i];
            // print.ln(header);
            raw.append(header.toString() + " ");
            if (header.getName().equalsIgnoreCase("Set-Cookie")) {
                setCookie.append(header.getValue());
                parseCookie(cookies, header.getValue());
            } else {
                // print.ln(header.getName()+"-"+header.getValue());
                Object value = responseHeader.get(KeyImpl.getInstance(header.getName()), null);
                if (value == null)
                    responseHeader.set(KeyImpl.getInstance(header.getName()), header.getValue());
                else {
                    Array arr = null;
                    if (value instanceof Array) {
                        arr = (Array) value;
                    } else {
                        arr = new ArrayImpl();
                        responseHeader.set(KeyImpl.getInstance(header.getName()), arr);
                        arr.appendEL(value);
                    }
                    arr.appendEL(header.getValue());
                }
            }
            // Content-Type
            if (header.getName().equalsIgnoreCase("Content-Type")) {
                mimetype = header.getValue();
                if (mimetype == null)
                    mimetype = NO_MIMETYPE;
            }
            // Content-Encoding
            if (header.getName().equalsIgnoreCase("Content-Encoding")) {
                contentEncoding = header.getValue();
            }
        }
        cfhttp.set(RESPONSEHEADER, responseHeader);
        cfhttp.set(KeyConstants._cookies, cookies);
        responseHeader.set(STATUS_CODE, new Double(statCode = rsp.getStatusCode()));
        responseHeader.set(EXPLANATION, (rsp.getStatusText()));
        if (setCookie.size() > 0)
            responseHeader.set(SET_COOKIE, setCookie);
        // is text
        boolean isText = mimetype == null || mimetype == NO_MIMETYPE || HTTPUtil.isTextMimeType(mimetype);
        // is multipart
        boolean isMultipart = MultiPartResponseUtils.isMultipart(mimetype);
        cfhttp.set(KeyConstants._text, Caster.toBoolean(isText));
        // boolean responseProvideCharset=false;
        if (!StringUtil.isEmpty(mimetype, true)) {
            if (isText) {
                String[] types = HTTPUtil.splitMimeTypeAndCharset(mimetype, null);
                if (types[0] != null)
                    cfhttp.set(KeyConstants._mimetype, types[0]);
                if (types[1] != null)
                    cfhttp.set(CHARSET, types[1]);
            } else
                cfhttp.set(KeyConstants._mimetype, mimetype);
        } else
            cfhttp.set(KeyConstants._mimetype, NO_MIMETYPE);
        // File
        Resource file = null;
        if (strFile != null && strPath != null) {
            file = ResourceUtil.toResourceNotExisting(pageContext, strPath).getRealResource(strFile);
        } else if (strFile != null) {
            file = ResourceUtil.toResourceNotExisting(pageContext, strFile);
        } else if (strPath != null) {
            file = ResourceUtil.toResourceNotExisting(pageContext, strPath);
            // Resource dir = file.getParentResource();
            if (file.isDirectory()) {
                // TODO was getName()
                file = file.getRealResource(req.getURI().getPath());
            // ->http://hc.apache.org/httpclient-3.x/apidocs/org/apache/commons/httpclient/URI.html#getName()
            }
        }
        if (file != null)
            pageContext.getConfig().getSecurityManager().checkFileLocation(file);
        // filecontent
        InputStream is = null;
        if (isText && getAsBinary != GET_AS_BINARY_YES) {
            String str;
            try {
                // read content
                if (method != METHOD_HEAD) {
                    is = rsp.getContentAsStream();
                    if (is != null && isGzipEncoded(contentEncoding))
                        is = rsp.getStatusCode() != 200 ? new CachingGZIPInputStream(is) : new GZIPInputStream(is);
                }
                try {
                    try {
                        str = is == null ? "" : IOUtil.toString(is, responseCharset, checkRemainingTimeout().getMillis());
                    } catch (EOFException eof) {
                        if (is instanceof CachingGZIPInputStream) {
                            str = IOUtil.toString(is = ((CachingGZIPInputStream) is).getRawData(), responseCharset, checkRemainingTimeout().getMillis());
                        } else
                            throw eof;
                    }
                } catch (UnsupportedEncodingException uee) {
                    str = IOUtil.toString(is, (Charset) null, checkRemainingTimeout().getMillis());
                }
            } catch (IOException ioe) {
                throw Caster.toPageException(ioe);
            } finally {
                IOUtil.closeEL(is);
            }
            if (str == null)
                str = "";
            if (resolveurl) {
                // if(e.redirectURL!=null)url=e.redirectURL.toExternalForm();
                str = new URLResolver().transform(str, e.response.getTargetURL(), false);
            }
            cfhttp.set(KeyConstants._filecontent, str);
            try {
                if (file != null) {
                    IOUtil.write(file, str, ((PageContextImpl) pageContext).getWebCharset(), false);
                }
            } catch (IOException e1) {
            }
            if (name != null) {
                Query qry = CSVParser.toQuery(str, delimiter, textqualifier, columns, firstrowasheaders);
                pageContext.setVariable(name, qry);
            }
        } else // Binary
        {
            byte[] barr = null;
            if (isGzipEncoded(contentEncoding)) {
                if (method != METHOD_HEAD) {
                    is = rsp.getContentAsStream();
                    is = rsp.getStatusCode() != 200 ? new CachingGZIPInputStream(is) : new GZIPInputStream(is);
                }
                try {
                    try {
                        barr = is == null ? new byte[0] : IOUtil.toBytes(is);
                    } catch (EOFException eof) {
                        if (is instanceof CachingGZIPInputStream)
                            barr = IOUtil.toBytes(((CachingGZIPInputStream) is).getRawData());
                        else
                            throw eof;
                    }
                } catch (IOException t) {
                    throw Caster.toPageException(t);
                } finally {
                    IOUtil.closeEL(is);
                }
            } else {
                try {
                    if (method != METHOD_HEAD)
                        barr = rsp.getContentAsByteArray();
                    else
                        barr = new byte[0];
                } catch (IOException t) {
                    throw Caster.toPageException(t);
                }
            }
            // IF Multipart response get file content and parse parts
            if (barr != null) {
                if (isMultipart) {
                    cfhttp.set(KeyConstants._filecontent, MultiPartResponseUtils.getParts(barr, mimetype));
                } else {
                    cfhttp.set(KeyConstants._filecontent, barr);
                }
            } else
                cfhttp.set(KeyConstants._filecontent, "");
            if (file != null) {
                try {
                    if (barr != null)
                        IOUtil.copy(new ByteArrayInputStream(barr), file, true);
                } catch (IOException ioe) {
                    throw Caster.toPageException(ioe);
                }
            }
        }
        // header
        cfhttp.set(KeyConstants._header, raw.toString());
        if (!isStatusOK(rsp.getStatusCode())) {
            String msg = rsp.getStatusCode() + " " + rsp.getStatusText();
            cfhttp.setEL(ERROR_DETAIL, msg);
            if (throwonerror) {
                throw new HTTPException(msg, null, rsp.getStatusCode(), rsp.getStatusText(), rsp.getURL());
            }
        }
        // TODO: check if we can use statCode instead of rsp.getStatusCode() everywhere and cleanup the code
        if (cacheHandler != null && rsp.getStatusCode() == 200) {
            // add to cache
            cacheHandler.set(pageContext, cacheId, cachedWithin, new HTTPCacheItem(cfhttp, url, System.nanoTime() - start));
        }
    } finally {
        if (client != null)
            client.close();
    }
}