Examples with TeeInputStream org.apache.commons.io.input.TeeInputStream used on opensource projects

Example 1 with TeeInputStream

use of org.apache.commons.io.input.TeeInputStream in project nifi by apache.

the class InvokeHTTP method onTrigger.

@Override
public void onTrigger(ProcessContext context, ProcessSession session) throws ProcessException {
    OkHttpClient okHttpClient = okHttpClientAtomicReference.get();
    FlowFile requestFlowFile = session.get();
    // Checking to see if the property to put the body of the response in an attribute was set
    boolean putToAttribute = context.getProperty(PROP_PUT_OUTPUT_IN_ATTRIBUTE).isSet();
    if (requestFlowFile == null) {
        if (context.hasNonLoopConnection()) {
            return;
        }
        String request = context.getProperty(PROP_METHOD).evaluateAttributeExpressions().getValue().toUpperCase();
        if ("POST".equals(request) || "PUT".equals(request) || "PATCH".equals(request)) {
            return;
        } else if (putToAttribute) {
            requestFlowFile = session.create();
        }
    }
    // Setting some initial variables
    final int maxAttributeSize = context.getProperty(PROP_PUT_ATTRIBUTE_MAX_LENGTH).asInteger();
    final ComponentLog logger = getLogger();
    // log ETag cache metrics
    final boolean eTagEnabled = context.getProperty(PROP_USE_ETAG).asBoolean();
    if (eTagEnabled && logger.isDebugEnabled()) {
        final Cache cache = okHttpClient.cache();
        logger.debug("OkHttp ETag cache metrics :: Request Count: {} | Network Count: {} | Hit Count: {}", new Object[] { cache.requestCount(), cache.networkCount(), cache.hitCount() });
    }
    // Every request/response cycle has a unique transaction id which will be stored as a flowfile attribute.
    final UUID txId = UUID.randomUUID();
    FlowFile responseFlowFile = null;
    try {
        // read the url property from the context
        final String urlstr = trimToEmpty(context.getProperty(PROP_URL).evaluateAttributeExpressions(requestFlowFile).getValue());
        final URL url = new URL(urlstr);
        Request httpRequest = configureRequest(context, session, requestFlowFile, url);
        // log request
        logRequest(logger, httpRequest);
        // emit send provenance event if successfully sent to the server
        if (httpRequest.body() != null) {
            session.getProvenanceReporter().send(requestFlowFile, url.toExternalForm(), true);
        }
        final long startNanos = System.nanoTime();
        Response responseHttp = okHttpClient.newCall(httpRequest).execute();
        // output the raw response headers (DEBUG level only)
        logResponse(logger, url, responseHttp);
        // store the status code and message
        int statusCode = responseHttp.code();
        String statusMessage = responseHttp.message();
        if (statusCode == 0) {
            throw new IllegalStateException("Status code unknown, connection hasn't been attempted.");
        }
        // Create a map of the status attributes that are always written to the request and response FlowFiles
        Map<String, String> statusAttributes = new HashMap<>();
        statusAttributes.put(STATUS_CODE, String.valueOf(statusCode));
        statusAttributes.put(STATUS_MESSAGE, statusMessage);
        statusAttributes.put(REQUEST_URL, url.toExternalForm());
        statusAttributes.put(TRANSACTION_ID, txId.toString());
        if (requestFlowFile != null) {
            requestFlowFile = session.putAllAttributes(requestFlowFile, statusAttributes);
        }
        // If the property to add the response headers to the request flowfile is true then add them
        if (context.getProperty(PROP_ADD_HEADERS_TO_REQUEST).asBoolean() && requestFlowFile != null) {
            // write the response headers as attributes
            // this will overwrite any existing flowfile attributes
            requestFlowFile = session.putAllAttributes(requestFlowFile, convertAttributesFromHeaders(url, responseHttp));
        }
        boolean outputBodyToRequestAttribute = (!isSuccess(statusCode) || putToAttribute) && requestFlowFile != null;
        boolean outputBodyToResponseContent = (isSuccess(statusCode) && !putToAttribute) || context.getProperty(PROP_OUTPUT_RESPONSE_REGARDLESS).asBoolean();
        ResponseBody responseBody = responseHttp.body();
        boolean bodyExists = responseBody != null;
        InputStream responseBodyStream = null;
        SoftLimitBoundedByteArrayOutputStream outputStreamToRequestAttribute = null;
        TeeInputStream teeInputStream = null;
        try {
            responseBodyStream = bodyExists ? responseBody.byteStream() : null;
            if (responseBodyStream != null && outputBodyToRequestAttribute && outputBodyToResponseContent) {
                outputStreamToRequestAttribute = new SoftLimitBoundedByteArrayOutputStream(maxAttributeSize);
                teeInputStream = new TeeInputStream(responseBodyStream, outputStreamToRequestAttribute);
            }
            if (outputBodyToResponseContent) {
                // clone the flowfile to capture the response
                if (requestFlowFile != null) {
                    responseFlowFile = session.create(requestFlowFile);
                } else {
                    responseFlowFile = session.create();
                }
                // write attributes to response flowfile
                responseFlowFile = session.putAllAttributes(responseFlowFile, statusAttributes);
                // write the response headers as attributes
                // this will overwrite any existing flowfile attributes
                responseFlowFile = session.putAllAttributes(responseFlowFile, convertAttributesFromHeaders(url, responseHttp));
                // can potentially be null in edge cases
                if (bodyExists) {
                    // write content type attribute to response flowfile if it is available
                    if (responseBody.contentType() != null) {
                        responseFlowFile = session.putAttribute(responseFlowFile, CoreAttributes.MIME_TYPE.key(), responseBody.contentType().toString());
                    }
                    if (teeInputStream != null) {
                        responseFlowFile = session.importFrom(teeInputStream, responseFlowFile);
                    } else {
                        responseFlowFile = session.importFrom(responseBodyStream, responseFlowFile);
                    }
                    // emit provenance event
                    final long millis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startNanos);
                    if (requestFlowFile != null) {
                        session.getProvenanceReporter().fetch(responseFlowFile, url.toExternalForm(), millis);
                    } else {
                        session.getProvenanceReporter().receive(responseFlowFile, url.toExternalForm(), millis);
                    }
                }
            }
            // if not successful and request flowfile is not null, store the response body into a flowfile attribute
            if (outputBodyToRequestAttribute && bodyExists) {
                String attributeKey = context.getProperty(PROP_PUT_OUTPUT_IN_ATTRIBUTE).evaluateAttributeExpressions(requestFlowFile).getValue();
                if (attributeKey == null) {
                    attributeKey = RESPONSE_BODY;
                }
                byte[] outputBuffer;
                int size;
                if (outputStreamToRequestAttribute != null) {
                    outputBuffer = outputStreamToRequestAttribute.getBuffer();
                    size = outputStreamToRequestAttribute.size();
                } else {
                    outputBuffer = new byte[maxAttributeSize];
                    size = StreamUtils.fillBuffer(responseBodyStream, outputBuffer, false);
                }
                String bodyString = new String(outputBuffer, 0, size, getCharsetFromMediaType(responseBody.contentType()));
                requestFlowFile = session.putAttribute(requestFlowFile, attributeKey, bodyString);
                final long millis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startNanos);
                session.getProvenanceReporter().modifyAttributes(requestFlowFile, "The " + attributeKey + " has been added. The value of which is the body of a http call to " + url.toExternalForm() + ". It took " + millis + "millis,");
            }
        } finally {
            if (outputStreamToRequestAttribute != null) {
                outputStreamToRequestAttribute.close();
                outputStreamToRequestAttribute = null;
            }
            if (teeInputStream != null) {
                teeInputStream.close();
                teeInputStream = null;
            } else if (responseBodyStream != null) {
                responseBodyStream.close();
                responseBodyStream = null;
            }
        }
        route(requestFlowFile, responseFlowFile, session, context, statusCode);
    } catch (final Exception e) {
        // penalize or yield
        if (requestFlowFile != null) {
            logger.error("Routing to {} due to exception: {}", new Object[] { REL_FAILURE.getName(), e }, e);
            requestFlowFile = session.penalize(requestFlowFile);
            requestFlowFile = session.putAttribute(requestFlowFile, EXCEPTION_CLASS, e.getClass().getName());
            requestFlowFile = session.putAttribute(requestFlowFile, EXCEPTION_MESSAGE, e.getMessage());
            // transfer original to failure
            session.transfer(requestFlowFile, REL_FAILURE);
        } else {
            logger.error("Yielding processor due to exception encountered as a source processor: {}", e);
            context.yield();
        }
        // cleanup response flowfile, if applicable
        try {
            if (responseFlowFile != null) {
                session.remove(responseFlowFile);
            }
        } catch (final Exception e1) {
            logger.error("Could not cleanup response flowfile due to exception: {}", new Object[] { e1 }, e1);
        }
    }
}

Example 2 with TeeInputStream

use of org.apache.commons.io.input.TeeInputStream in project wso2-axis2-transports by wso2.

the class LogAspect method aroundProcessDocument.

@Around("call(org.apache.axiom.om.OMElement org.apache.axis2.builder.Builder.processDocument(" + "       java.io.InputStream, java.lang.String, org.apache.axis2.context.MessageContext))" + " && args(in, contentType, msgContext)")
public Object aroundProcessDocument(ProceedingJoinPoint proceedingJoinPoint, InputStream in, String contentType, MessageContext msgContext) throws Throwable {
    InputStream tee;
    if (in == null) {
        tee = null;
    } else {
        OutputStream log = LogManager.INSTANCE.createLog("builder");
        // Note: We can't close the log right after the method execution because the
        // message builder may use streaming. LogManager will take care of closing the
        // log for us if anything goes wrong.
        tee = new TeeInputStream(in, log, true);
    }
    return proceedingJoinPoint.proceed(new Object[] { tee, contentType, msgContext });
}

Example 3 with TeeInputStream

use of org.apache.commons.io.input.TeeInputStream in project nifi-minifi by apache.

the class FileChangeIngestor method run.

@Override
public void run() {
    logger.debug("Checking for a change");
    if (targetChanged()) {
        logger.debug("Target changed, checking if it's different than current flow.");
        try (FileInputStream configFile = new FileInputStream(configFilePath.toFile());
            ByteArrayOutputStream pipedOutputStream = new ByteArrayOutputStream();
            TeeInputStream teeInputStream = new TeeInputStream(configFile, pipedOutputStream)) {
            if (differentiator.isNew(teeInputStream)) {
                logger.debug("New change, notifying listener");
                // Fill the byteArrayOutputStream with the rest of the request data
                while (teeInputStream.available() != 0) {
                    teeInputStream.read();
                }
                ByteBuffer newConfig = ByteBuffer.wrap(pipedOutputStream.toByteArray());
                ByteBuffer readOnlyNewConfig = newConfig.asReadOnlyBuffer();
                configurationChangeNotifier.notifyListeners(readOnlyNewConfig);
                logger.debug("Listeners notified");
            }
        } catch (Exception e) {
            logger.error("Could not successfully notify listeners.", e);
        }
    }
}

Example 4 with TeeInputStream

use of org.apache.commons.io.input.TeeInputStream in project application by collectionspace.

the class NullResolver method serveSingle.

/**
 * Return a single request from the cspace-ui servlet context
 * @param tenant
 * @param sub
 * @param sc
 * @throws UIException
 * @throws IOException
 */
private void serveSingle(String tenant, CompositeWebUIRequestPart sub, ServletContext sc) throws UIException, IOException {
    // Setup our request object
    UI web = tenantCSPM.get(tenant).getUI("web");
    WebUI webui = (WebUI) web;
    UIMapping[] allmappings = webui.getAllMappings();
    ConfigRoot root = tenantCSPM.get(tenant).getConfigRoot();
    Spec spec = (Spec) root.getRoot(Spec.SPEC_ROOT);
    // this doesn't seem to work yet
    Boolean doMetaConfig = true;
    String[] pathbits = sub.getPrincipalPath();
    String pathinfo = StringUtils.join(pathbits, "/");
    String path = pathinfo;
    Map<String, Object> validmap = doMapping(pathinfo, allmappings, spec);
    if (null != validmap) {
        path = (String) validmap.get("File");
        if (((UIMapping) validmap.get("map")).hasMetaConfig() && doMetaConfig) {
            InputStream is = getFixedContent(sc, path, tenant);
            StringWriter writer = new StringWriter();
            IOUtils.copy(is, writer, "UTF-8");
            String theString = writer.toString();
            /*
				SAXReader reader = new SAXReader();
				reader.setEntityResolver(new NullResolver());
				Document xmlfile = null;

				ByteArrayOutputStream dump = new ByteArrayOutputStream();
				xmlfile = reader.read(new TeeInputStream(is, dump));
			//	log.info(dump.toString("UTF-8"));
			//	log.info(xmlfile.asXML());<tr></tr<.>
			 
			 TODO I want this to work... it just doesn't yet as I haven\'t had time to work on it
			 this is all about replace items in the title etc of the universal files so they become more specific
*/
            for (String metafield : ((UIMapping) validmap.get("map")).getAllMetaConfigs()) {
                String rtext = ((UIMapping) validmap.get("map")).getMetaConfig(metafield).parseValue((Record) validmap.get("record"), (Instance) validmap.get("instance"));
                // try as json
                String regex = "\"" + metafield + "\": \"[^\"]*\"";
                String replacement = "\"" + metafield + "\": \"" + rtext + "\"";
                theString = theString.replaceFirst(regex, replacement);
                // try as xml
                String regex2 = "<" + metafield + ">[^<]*</" + metafield + ">";
                String replacement2 = "<" + metafield + ">" + rtext + "</" + metafield + ">";
                theString = theString.replaceFirst(regex2, replacement2);
            }
            InputStream is2 = new ByteArrayInputStream(theString.getBytes("UTF-8"));
            serveExternalContent(sub, sc, is2, path);
            is.close();
            return;
        }
    }
    if (pathbits[1].equals("bundle")) {
        if (serverCreateMergedExternalContent(sub, sc, pathinfo, tenant)) {
            return;
        }
    }
    if (serverFixedExternalContent(sub, sc)) {
        return;
    }
}

Example 5 with TeeInputStream

use of org.apache.commons.io.input.TeeInputStream in project application by collectionspace.

the class NullResolver method serveSingle.

/**
 * abstraction for returning a single item.
 * probably could be simplified with other serveSingle function
 * @param tenant
 * @param servlet_request
 * @param servlet_response
 * @param pathinfo
 * @param pathbits
 * @param sc
 * @throws IOException
 * @throws BadRequestException
 * @throws UnsupportedEncodingException
 */
private void serveSingle(String tenant, HttpServletRequest servlet_request, HttpServletResponse servlet_response, String pathinfo, String[] pathbits, ServletContext sc) throws IOException, BadRequestException, UnsupportedEncodingException {
    // should we redirect this url or just do the normal stuff
    // Setup our request object
    UI web = tenantCSPM.get(tenant).getUI("web");
    WebUI webui = (WebUI) web;
    UIMapping[] allmappings = webui.getAllMappings();
    ConfigRoot root = tenantCSPM.get(tenant).getConfigRoot();
    Spec spec = (Spec) root.getRoot(Spec.SPEC_ROOT);
    // this doesn't seem to work yet
    Boolean doMetaConfig = true;
    String path = pathinfo;
    Map<String, Object> validmap = doMapping(pathinfo, allmappings, spec);
    if (null != validmap) {
        path = (String) validmap.get("File");
        if (((UIMapping) validmap.get("map")).hasMetaConfig() && doMetaConfig) {
            InputStream is = getFixedContent(sc, path, tenant);
            StringWriter writer = new StringWriter();
            IOUtils.copy(is, writer, "UTF-8");
            String theString = writer.toString();
            /*
				SAXReader reader = new SAXReader();
				reader.setEntityResolver(new NullResolver());
				Document xmlfile = null;

				ByteArrayOutputStream dump = new ByteArrayOutputStream();
				xmlfile = reader.read(new TeeInputStream(is, dump));
			//	log.info(dump.toString("UTF-8"));
			//	log.info(xmlfile.asXML());<tr></tr<.>
			 TODO I want this to work... it just doesn't yet as I haven\'t had time to work on it
			 this is all about replace items in the title etc of the universal files so they become more specific
*/
            for (String metafield : ((UIMapping) validmap.get("map")).getAllMetaConfigs()) {
                String rtext = ((UIMapping) validmap.get("map")).getMetaConfig(metafield).parseValue((Record) validmap.get("record"), (Instance) validmap.get("instance"));
                // try as json
                String regex = "\"" + metafield + "\": \"[^\"]*\"";
                String replacement = "\"" + metafield + "\": \"" + rtext + "\"";
                theString = theString.replaceFirst(regex, replacement);
                // try as xml
                String regex2 = "<" + metafield + ">[^<]*</" + metafield + ">";
                String replacement2 = "<" + metafield + ">" + rtext + "</" + metafield + ">";
                theString = theString.replaceFirst(regex2, replacement2);
            }
            InputStream is2 = new ByteArrayInputStream(theString.getBytes("UTF-8"));
            serveContent(servlet_response, is2);
            is.close();
            return;
        }
    }
    if (pathbits[1].equals("bundle")) {
        if (serverCreateMergedExternalContent(servlet_request, servlet_response, sc, pathinfo, tenant)) {
            return;
        }
    }
    if (serverFixedExternalContent(servlet_request, servlet_response, sc, path, tenant)) {
        return;
    }
}