Examples with ProcessException org.apache.nifi.processor.exception.ProcessException used on opensource projects

Example 61 with ProcessException

use of org.apache.nifi.processor.exception.ProcessException in project nifi by apache.

the class UnpackContent method onTrigger.

@Override
public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException {
    FlowFile flowFile = session.get();
    if (flowFile == null) {
        return;
    }
    final ComponentLog logger = getLogger();
    PackageFormat packagingFormat = PackageFormat.getFormat(context.getProperty(PACKAGING_FORMAT).getValue().toLowerCase());
    if (packagingFormat == PackageFormat.AUTO_DETECT_FORMAT) {
        packagingFormat = null;
        final String mimeType = flowFile.getAttribute(CoreAttributes.MIME_TYPE.key());
        if (mimeType == null) {
            logger.error("No mime.type attribute set for {}; routing to failure", new Object[] { flowFile });
            session.transfer(flowFile, REL_FAILURE);
            return;
        }
        for (PackageFormat format : PackageFormat.values()) {
            if (mimeType.toLowerCase().equals(format.getMimeType())) {
                packagingFormat = format;
            }
        }
        if (packagingFormat == null) {
            logger.info("Cannot unpack {} because its mime.type attribute is set to '{}', which is not a format that can be unpacked; routing to 'success'", new Object[] { flowFile, mimeType });
            session.transfer(flowFile, REL_SUCCESS);
            return;
        }
    }
    // set the Unpacker to use for this FlowFile.  FlowFileUnpackager objects maintain state and are not reusable.
    final Unpacker unpacker;
    final boolean addFragmentAttrs;
    switch(packagingFormat) {
        case TAR_FORMAT:
        case X_TAR_FORMAT:
            unpacker = tarUnpacker;
            addFragmentAttrs = true;
            break;
        case ZIP_FORMAT:
            unpacker = zipUnpacker;
            addFragmentAttrs = true;
            break;
        case FLOWFILE_STREAM_FORMAT_V2:
            unpacker = new FlowFileStreamUnpacker(new FlowFileUnpackagerV2());
            addFragmentAttrs = false;
            break;
        case FLOWFILE_STREAM_FORMAT_V3:
            unpacker = new FlowFileStreamUnpacker(new FlowFileUnpackagerV3());
            addFragmentAttrs = false;
            break;
        case FLOWFILE_TAR_FORMAT:
            unpacker = new FlowFileStreamUnpacker(new FlowFileUnpackagerV1());
            addFragmentAttrs = false;
            break;
        case AUTO_DETECT_FORMAT:
        default:
            // The format of the unpacker should be known before initialization
            throw new ProcessException(packagingFormat + " is not a valid packaging format");
    }
    final List<FlowFile> unpacked = new ArrayList<>();
    try {
        unpacker.unpack(session, flowFile, unpacked);
        if (unpacked.isEmpty()) {
            logger.error("Unable to unpack {} because it does not appear to have any entries; routing to failure", new Object[] { flowFile });
            session.transfer(flowFile, REL_FAILURE);
            return;
        }
        if (addFragmentAttrs) {
            finishFragmentAttributes(session, flowFile, unpacked);
        }
        session.transfer(unpacked, REL_SUCCESS);
        final String fragmentId = unpacked.size() > 0 ? unpacked.get(0).getAttribute(FRAGMENT_ID) : null;
        flowFile = FragmentAttributes.copyAttributesToOriginal(session, flowFile, fragmentId, unpacked.size());
        session.transfer(flowFile, REL_ORIGINAL);
        session.getProvenanceReporter().fork(flowFile, unpacked);
        logger.info("Unpacked {} into {} and transferred to success", new Object[] { flowFile, unpacked });
    } catch (final ProcessException | InvalidPathException e) {
        logger.error("Unable to unpack {} due to {}; routing to failure", new Object[] { flowFile, e });
        session.transfer(flowFile, REL_FAILURE);
        session.remove(unpacked);
    }
}

Example 62 with ProcessException

use of org.apache.nifi.processor.exception.ProcessException in project nifi by apache.

the class GetHTTP method updateStateMap.

private void updateStateMap(ProcessContext context, HttpResponse response, StateMap beforeStateMap, String url) {
    try {
        Map<String, String> workingMap = new HashMap<>();
        workingMap.putAll(beforeStateMap.toMap());
        final StateManager stateManager = context.getStateManager();
        StateMap oldValue = beforeStateMap;
        long currentTime = System.currentTimeMillis();
        final Header receivedLastModified = response.getFirstHeader(HEADER_LAST_MODIFIED);
        if (receivedLastModified != null) {
            workingMap.put(LAST_MODIFIED + ":" + url, currentTime + ":" + receivedLastModified.getValue());
        }
        final Header receivedEtag = response.getFirstHeader(HEADER_ETAG);
        if (receivedEtag != null) {
            workingMap.put(ETAG + ":" + url, currentTime + ":" + receivedEtag.getValue());
        }
        boolean replaceSucceeded = stateManager.replace(oldValue, workingMap, Scope.LOCAL);
        boolean changed;
        while (!replaceSucceeded) {
            oldValue = stateManager.getState(Scope.LOCAL);
            workingMap.clear();
            workingMap.putAll(oldValue.toMap());
            changed = false;
            if (receivedLastModified != null) {
                Tuple<String, String> storedLastModifiedTuple = parseStateValue(workingMap.get(LAST_MODIFIED + ":" + url));
                if (Long.parseLong(storedLastModifiedTuple.getKey()) < currentTime) {
                    workingMap.put(LAST_MODIFIED + ":" + url, currentTime + ":" + receivedLastModified.getValue());
                    changed = true;
                }
            }
            if (receivedEtag != null) {
                Tuple<String, String> storedLastModifiedTuple = parseStateValue(workingMap.get(ETAG + ":" + url));
                if (Long.parseLong(storedLastModifiedTuple.getKey()) < currentTime) {
                    workingMap.put(ETAG + ":" + url, currentTime + ":" + receivedEtag.getValue());
                    changed = true;
                }
            }
            if (changed) {
                replaceSucceeded = stateManager.replace(oldValue, workingMap, Scope.LOCAL);
            } else {
                break;
            }
        }
    } catch (final IOException ioe) {
        throw new ProcessException(ioe);
    }
}

Example 63 with ProcessException

use of org.apache.nifi.processor.exception.ProcessException in project nifi by apache.

the class HandleHttpResponse method onTrigger.

@Override
public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException {
    FlowFile flowFile = session.get();
    if (flowFile == null) {
        return;
    }
    final StopWatch stopWatch = new StopWatch(true);
    final String contextIdentifier = flowFile.getAttribute(HTTPUtils.HTTP_CONTEXT_ID);
    if (contextIdentifier == null) {
        session.transfer(flowFile, REL_FAILURE);
        getLogger().warn("Failed to respond to HTTP request for {} because FlowFile did not have an '" + HTTPUtils.HTTP_CONTEXT_ID + "' attribute", new Object[] { flowFile });
        return;
    }
    final String statusCodeValue = context.getProperty(STATUS_CODE).evaluateAttributeExpressions(flowFile).getValue();
    if (!isNumber(statusCodeValue)) {
        session.transfer(flowFile, REL_FAILURE);
        getLogger().error("Failed to respond to HTTP request for {} because status code was '{}', which is not a valid number", new Object[] { flowFile, statusCodeValue });
        return;
    }
    final HttpContextMap contextMap = context.getProperty(HTTP_CONTEXT_MAP).asControllerService(HttpContextMap.class);
    final HttpServletResponse response = contextMap.getResponse(contextIdentifier);
    if (response == null) {
        session.transfer(flowFile, REL_FAILURE);
        getLogger().error("Failed to respond to HTTP request for {} because FlowFile had an '{}' attribute of {} but could not find an HTTP Response Object for this identifier", new Object[] { flowFile, HTTPUtils.HTTP_CONTEXT_ID, contextIdentifier });
        return;
    }
    final int statusCode = Integer.parseInt(statusCodeValue);
    response.setStatus(statusCode);
    for (final Map.Entry<PropertyDescriptor, String> entry : context.getProperties().entrySet()) {
        final PropertyDescriptor descriptor = entry.getKey();
        if (descriptor.isDynamic()) {
            final String headerName = descriptor.getName();
            final String headerValue = context.getProperty(descriptor).evaluateAttributeExpressions(flowFile).getValue();
            if (!headerValue.trim().isEmpty()) {
                response.setHeader(headerName, headerValue);
            }
        }
    }
    try {
        session.exportTo(flowFile, response.getOutputStream());
        response.flushBuffer();
    } catch (final ProcessException e) {
        session.transfer(flowFile, REL_FAILURE);
        getLogger().error("Failed to respond to HTTP request for {} due to {}", new Object[] { flowFile, e });
        contextMap.complete(contextIdentifier);
        return;
    } catch (final Exception e) {
        session.transfer(flowFile, REL_FAILURE);
        getLogger().error("Failed to respond to HTTP request for {} due to {}", new Object[] { flowFile, e });
        return;
    }
    try {
        contextMap.complete(contextIdentifier);
    } catch (final IllegalStateException ise) {
        getLogger().error("Failed to complete HTTP Transaction for {} due to {}", new Object[] { flowFile, ise });
        session.transfer(flowFile, REL_FAILURE);
        return;
    }
    session.getProvenanceReporter().send(flowFile, HTTPUtils.getURI(flowFile.getAttributes()), stopWatch.getElapsed(TimeUnit.MILLISECONDS));
    session.transfer(flowFile, REL_SUCCESS);
    getLogger().info("Successfully responded to HTTP Request for {} with status code {}", new Object[] { flowFile, statusCode });
}

Example 64 with ProcessException

use of org.apache.nifi.processor.exception.ProcessException in project nifi by apache.

the class InvokeHTTP method onTrigger.

@Override
public void onTrigger(ProcessContext context, ProcessSession session) throws ProcessException {
    OkHttpClient okHttpClient = okHttpClientAtomicReference.get();
    FlowFile requestFlowFile = session.get();
    // Checking to see if the property to put the body of the response in an attribute was set
    boolean putToAttribute = context.getProperty(PROP_PUT_OUTPUT_IN_ATTRIBUTE).isSet();
    if (requestFlowFile == null) {
        if (context.hasNonLoopConnection()) {
            return;
        }
        String request = context.getProperty(PROP_METHOD).evaluateAttributeExpressions().getValue().toUpperCase();
        if ("POST".equals(request) || "PUT".equals(request) || "PATCH".equals(request)) {
            return;
        } else if (putToAttribute) {
            requestFlowFile = session.create();
        }
    }
    // Setting some initial variables
    final int maxAttributeSize = context.getProperty(PROP_PUT_ATTRIBUTE_MAX_LENGTH).asInteger();
    final ComponentLog logger = getLogger();
    // log ETag cache metrics
    final boolean eTagEnabled = context.getProperty(PROP_USE_ETAG).asBoolean();
    if (eTagEnabled && logger.isDebugEnabled()) {
        final Cache cache = okHttpClient.cache();
        logger.debug("OkHttp ETag cache metrics :: Request Count: {} | Network Count: {} | Hit Count: {}", new Object[] { cache.requestCount(), cache.networkCount(), cache.hitCount() });
    }
    // Every request/response cycle has a unique transaction id which will be stored as a flowfile attribute.
    final UUID txId = UUID.randomUUID();
    FlowFile responseFlowFile = null;
    try {
        // read the url property from the context
        final String urlstr = trimToEmpty(context.getProperty(PROP_URL).evaluateAttributeExpressions(requestFlowFile).getValue());
        final URL url = new URL(urlstr);
        Request httpRequest = configureRequest(context, session, requestFlowFile, url);
        // log request
        logRequest(logger, httpRequest);
        // emit send provenance event if successfully sent to the server
        if (httpRequest.body() != null) {
            session.getProvenanceReporter().send(requestFlowFile, url.toExternalForm(), true);
        }
        final long startNanos = System.nanoTime();
        Response responseHttp = okHttpClient.newCall(httpRequest).execute();
        // output the raw response headers (DEBUG level only)
        logResponse(logger, url, responseHttp);
        // store the status code and message
        int statusCode = responseHttp.code();
        String statusMessage = responseHttp.message();
        if (statusCode == 0) {
            throw new IllegalStateException("Status code unknown, connection hasn't been attempted.");
        }
        // Create a map of the status attributes that are always written to the request and response FlowFiles
        Map<String, String> statusAttributes = new HashMap<>();
        statusAttributes.put(STATUS_CODE, String.valueOf(statusCode));
        statusAttributes.put(STATUS_MESSAGE, statusMessage);
        statusAttributes.put(REQUEST_URL, url.toExternalForm());
        statusAttributes.put(TRANSACTION_ID, txId.toString());
        if (requestFlowFile != null) {
            requestFlowFile = session.putAllAttributes(requestFlowFile, statusAttributes);
        }
        // If the property to add the response headers to the request flowfile is true then add them
        if (context.getProperty(PROP_ADD_HEADERS_TO_REQUEST).asBoolean() && requestFlowFile != null) {
            // write the response headers as attributes
            // this will overwrite any existing flowfile attributes
            requestFlowFile = session.putAllAttributes(requestFlowFile, convertAttributesFromHeaders(url, responseHttp));
        }
        boolean outputBodyToRequestAttribute = (!isSuccess(statusCode) || putToAttribute) && requestFlowFile != null;
        boolean outputBodyToResponseContent = (isSuccess(statusCode) && !putToAttribute) || context.getProperty(PROP_OUTPUT_RESPONSE_REGARDLESS).asBoolean();
        ResponseBody responseBody = responseHttp.body();
        boolean bodyExists = responseBody != null;
        InputStream responseBodyStream = null;
        SoftLimitBoundedByteArrayOutputStream outputStreamToRequestAttribute = null;
        TeeInputStream teeInputStream = null;
        try {
            responseBodyStream = bodyExists ? responseBody.byteStream() : null;
            if (responseBodyStream != null && outputBodyToRequestAttribute && outputBodyToResponseContent) {
                outputStreamToRequestAttribute = new SoftLimitBoundedByteArrayOutputStream(maxAttributeSize);
                teeInputStream = new TeeInputStream(responseBodyStream, outputStreamToRequestAttribute);
            }
            if (outputBodyToResponseContent) {
                // clone the flowfile to capture the response
                if (requestFlowFile != null) {
                    responseFlowFile = session.create(requestFlowFile);
                } else {
                    responseFlowFile = session.create();
                }
                // write attributes to response flowfile
                responseFlowFile = session.putAllAttributes(responseFlowFile, statusAttributes);
                // write the response headers as attributes
                // this will overwrite any existing flowfile attributes
                responseFlowFile = session.putAllAttributes(responseFlowFile, convertAttributesFromHeaders(url, responseHttp));
                // can potentially be null in edge cases
                if (bodyExists) {
                    // write content type attribute to response flowfile if it is available
                    if (responseBody.contentType() != null) {
                        responseFlowFile = session.putAttribute(responseFlowFile, CoreAttributes.MIME_TYPE.key(), responseBody.contentType().toString());
                    }
                    if (teeInputStream != null) {
                        responseFlowFile = session.importFrom(teeInputStream, responseFlowFile);
                    } else {
                        responseFlowFile = session.importFrom(responseBodyStream, responseFlowFile);
                    }
                    // emit provenance event
                    final long millis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startNanos);
                    if (requestFlowFile != null) {
                        session.getProvenanceReporter().fetch(responseFlowFile, url.toExternalForm(), millis);
                    } else {
                        session.getProvenanceReporter().receive(responseFlowFile, url.toExternalForm(), millis);
                    }
                }
            }
            // if not successful and request flowfile is not null, store the response body into a flowfile attribute
            if (outputBodyToRequestAttribute && bodyExists) {
                String attributeKey = context.getProperty(PROP_PUT_OUTPUT_IN_ATTRIBUTE).evaluateAttributeExpressions(requestFlowFile).getValue();
                if (attributeKey == null) {
                    attributeKey = RESPONSE_BODY;
                }
                byte[] outputBuffer;
                int size;
                if (outputStreamToRequestAttribute != null) {
                    outputBuffer = outputStreamToRequestAttribute.getBuffer();
                    size = outputStreamToRequestAttribute.size();
                } else {
                    outputBuffer = new byte[maxAttributeSize];
                    size = StreamUtils.fillBuffer(responseBodyStream, outputBuffer, false);
                }
                String bodyString = new String(outputBuffer, 0, size, getCharsetFromMediaType(responseBody.contentType()));
                requestFlowFile = session.putAttribute(requestFlowFile, attributeKey, bodyString);
                final long millis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startNanos);
                session.getProvenanceReporter().modifyAttributes(requestFlowFile, "The " + attributeKey + " has been added. The value of which is the body of a http call to " + url.toExternalForm() + ". It took " + millis + "millis,");
            }
        } finally {
            if (outputStreamToRequestAttribute != null) {
                outputStreamToRequestAttribute.close();
                outputStreamToRequestAttribute = null;
            }
            if (teeInputStream != null) {
                teeInputStream.close();
                teeInputStream = null;
            } else if (responseBodyStream != null) {
                responseBodyStream.close();
                responseBodyStream = null;
            }
        }
        route(requestFlowFile, responseFlowFile, session, context, statusCode);
    } catch (final Exception e) {
        // penalize or yield
        if (requestFlowFile != null) {
            logger.error("Routing to {} due to exception: {}", new Object[] { REL_FAILURE.getName(), e }, e);
            requestFlowFile = session.penalize(requestFlowFile);
            requestFlowFile = session.putAttribute(requestFlowFile, EXCEPTION_CLASS, e.getClass().getName());
            requestFlowFile = session.putAttribute(requestFlowFile, EXCEPTION_MESSAGE, e.getMessage());
            // transfer original to failure
            session.transfer(requestFlowFile, REL_FAILURE);
        } else {
            logger.error("Yielding processor due to exception encountered as a source processor: {}", e);
            context.yield();
        }
        // cleanup response flowfile, if applicable
        try {
            if (responseFlowFile != null) {
                session.remove(responseFlowFile);
            }
        } catch (final Exception e1) {
            logger.error("Could not cleanup response flowfile due to exception: {}", new Object[] { e1 }, e1);
        }
    }
}

Example 65 with ProcessException

use of org.apache.nifi.processor.exception.ProcessException in project nifi by apache.

the class JmsConsumer method map2FlowFile.

public static JmsProcessingSummary map2FlowFile(final ProcessContext context, final ProcessSession session, final Message message, final boolean addAttributes, ComponentLog logger) throws Exception {
    // Currently not very useful, because always one Message == one FlowFile
    final AtomicInteger msgsThisFlowFile = new AtomicInteger(1);
    FlowFile flowFile = session.create();
    try {
        // MapMessage is exception, add only name-value pairs to FlowFile attributes
        if (message instanceof MapMessage) {
            MapMessage mapMessage = (MapMessage) message;
            flowFile = session.putAllAttributes(flowFile, createMapMessageValues(mapMessage));
        } else {
            // all other message types, write Message body to FlowFile content
            flowFile = session.write(flowFile, new OutputStreamCallback() {

                @Override
                public void process(final OutputStream rawOut) throws IOException {
                    try (final OutputStream out = new BufferedOutputStream(rawOut, 65536)) {
                        final byte[] messageBody = JmsFactory.createByteArray(message);
                        out.write(messageBody);
                    } catch (final JMSException e) {
                        throw new ProcessException("Failed to receive JMS Message due to " + e.getMessage(), e);
                    }
                }
            });
        }
        if (addAttributes) {
            flowFile = session.putAllAttributes(flowFile, JmsFactory.createAttributeMap(message));
        }
        session.getProvenanceReporter().receive(flowFile, context.getProperty(URL).getValue());
        session.transfer(flowFile, REL_SUCCESS);
        logger.info("Created {} from {} messages received from JMS Server and transferred to 'success'", new Object[] { flowFile, msgsThisFlowFile.get() });
        return new JmsProcessingSummary(flowFile.getSize(), message, flowFile);
    } catch (Exception e) {
        session.remove(flowFile);
        throw e;
    }
}