Examples with ProcessSession org.apache.nifi.processor.ProcessSession used on opensource projects

Example 81 with ProcessSession

use of org.apache.nifi.processor.ProcessSession in project nifi by apache.

the class DeleteRowsWriter method writeEvent.

/**
 * Creates and transfers a new flow file whose contents are the JSON-serialized value of the specified event, and the sequence ID attribute set
 *
 * @param session   A reference to a ProcessSession from which the flow file(s) will be created and transferred
 * @param eventInfo An event whose value will become the contents of the flow file
 * @return The next available CDC sequence ID for use by the CDC processor
 */
@Override
public long writeEvent(final ProcessSession session, String transitUri, final DeleteRowsEventInfo eventInfo, final long currentSequenceId, Relationship relationship) {
    final AtomicLong seqId = new AtomicLong(currentSequenceId);
    for (Serializable[] row : eventInfo.getRows()) {
        FlowFile flowFile = session.create();
        flowFile = session.write(flowFile, outputStream -> {
            super.startJson(outputStream, eventInfo);
            super.writeJson(eventInfo);
            final BitSet bitSet = eventInfo.getIncludedColumns();
            writeRow(eventInfo, row, bitSet);
            super.endJson();
        });
        flowFile = session.putAllAttributes(flowFile, getCommonAttributes(seqId.get(), eventInfo));
        session.transfer(flowFile, relationship);
        session.getProvenanceReporter().receive(flowFile, transitUri);
        seqId.getAndIncrement();
    }
    return seqId.get();
}

Example 82 with ProcessSession

use of org.apache.nifi.processor.ProcessSession in project nifi by apache.

the class InsertRowsWriter method writeEvent.

/**
 * Creates and transfers a new flow file whose contents are the JSON-serialized value of the specified event, and the sequence ID attribute set
 *
 * @param session   A reference to a ProcessSession from which the flow file(s) will be created and transferred
 * @param eventInfo An event whose value will become the contents of the flow file
 * @return The next available CDC sequence ID for use by the CDC processor
 */
@Override
public long writeEvent(final ProcessSession session, String transitUri, final InsertRowsEventInfo eventInfo, final long currentSequenceId, Relationship relationship) {
    final AtomicLong seqId = new AtomicLong(currentSequenceId);
    for (Serializable[] row : eventInfo.getRows()) {
        FlowFile flowFile = session.create();
        flowFile = session.write(flowFile, outputStream -> {
            super.startJson(outputStream, eventInfo);
            super.writeJson(eventInfo);
            final BitSet bitSet = eventInfo.getIncludedColumns();
            writeRow(eventInfo, row, bitSet);
            super.endJson();
        });
        flowFile = session.putAllAttributes(flowFile, getCommonAttributes(seqId.get(), eventInfo));
        session.transfer(flowFile, relationship);
        session.getProvenanceReporter().receive(flowFile, transitUri);
        seqId.getAndIncrement();
    }
    return seqId.get();
}

Example 83 with ProcessSession

use of org.apache.nifi.processor.ProcessSession in project nifi by apache.

the class PutDruidRecord method onTrigger.

public void onTrigger(ProcessContext context, ProcessSessionFactory factory) throws ProcessException {
    final ProcessSession session = factory.createSession();
    processFlowFile(context, session);
}

Example 84 with ProcessSession

use of org.apache.nifi.processor.ProcessSession in project nifi by apache.

the class PutElasticsearchHttp method onTrigger.

@Override
public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException {
    final int batchSize = context.getProperty(BATCH_SIZE).evaluateAttributeExpressions().asInteger();
    final List<FlowFile> flowFiles = session.get(batchSize);
    if (flowFiles.isEmpty()) {
        return;
    }
    final String id_attribute = context.getProperty(ID_ATTRIBUTE).getValue();
    // Authentication
    final String username = context.getProperty(USERNAME).evaluateAttributeExpressions().getValue();
    final String password = context.getProperty(PASSWORD).evaluateAttributeExpressions().getValue();
    OkHttpClient okHttpClient = getClient();
    final ComponentLog logger = getLogger();
    // Keep track of the list of flow files that need to be transferred. As they are transferred, remove them from the list.
    List<FlowFile> flowFilesToTransfer = new LinkedList<>(flowFiles);
    final StringBuilder sb = new StringBuilder();
    final String baseUrl = trimToEmpty(context.getProperty(ES_URL).evaluateAttributeExpressions().getValue());
    HttpUrl.Builder urlBuilder = HttpUrl.parse(baseUrl).newBuilder().addPathSegment("_bulk");
    // Find the user-added properties and set them as query parameters on the URL
    for (Map.Entry<PropertyDescriptor, String> property : context.getProperties().entrySet()) {
        PropertyDescriptor pd = property.getKey();
        if (pd.isDynamic()) {
            if (property.getValue() != null) {
                urlBuilder = urlBuilder.addQueryParameter(pd.getName(), context.getProperty(pd).evaluateAttributeExpressions().getValue());
            }
        }
    }
    final URL url = urlBuilder.build().url();
    for (FlowFile file : flowFiles) {
        final String index = context.getProperty(INDEX).evaluateAttributeExpressions(file).getValue();
        final Charset charset = Charset.forName(context.getProperty(CHARSET).evaluateAttributeExpressions(file).getValue());
        if (StringUtils.isEmpty(index)) {
            logger.error("No value for index in for {}, transferring to failure", new Object[] { id_attribute, file });
            flowFilesToTransfer.remove(file);
            session.transfer(file, REL_FAILURE);
            continue;
        }
        final String docType = context.getProperty(TYPE).evaluateAttributeExpressions(file).getValue();
        String indexOp = context.getProperty(INDEX_OP).evaluateAttributeExpressions(file).getValue();
        if (StringUtils.isEmpty(indexOp)) {
            logger.error("No Index operation specified for {}, transferring to failure.", new Object[] { file });
            flowFilesToTransfer.remove(file);
            session.transfer(file, REL_FAILURE);
            continue;
        }
        switch(indexOp.toLowerCase()) {
            case "index":
            case "update":
            case "upsert":
            case "delete":
                break;
            default:
                logger.error("Index operation {} not supported for {}, transferring to failure.", new Object[] { indexOp, file });
                flowFilesToTransfer.remove(file);
                session.transfer(file, REL_FAILURE);
                continue;
        }
        final String id = (id_attribute != null) ? file.getAttribute(id_attribute) : null;
        // a missing ID indicates one is to be auto-generated by Elasticsearch
        if (id == null && !indexOp.equalsIgnoreCase("index")) {
            logger.error("Index operation {} requires a valid identifier value from a flow file attribute, transferring to failure.", new Object[] { indexOp, file });
            flowFilesToTransfer.remove(file);
            session.transfer(file, REL_FAILURE);
            continue;
        }
        final StringBuilder json = new StringBuilder();
        session.read(file, in -> {
            json.append(IOUtils.toString(in, charset).replace("\r\n", " ").replace('\n', ' ').replace('\r', ' '));
        });
        if (indexOp.equalsIgnoreCase("index")) {
            sb.append("{\"index\": { \"_index\": \"");
            sb.append(index);
            sb.append("\", \"_type\": \"");
            sb.append(docType);
            sb.append("\"");
            if (!StringUtils.isEmpty(id)) {
                sb.append(", \"_id\": \"");
                sb.append(id);
                sb.append("\"");
            }
            sb.append("}}\n");
            sb.append(json);
            sb.append("\n");
        } else if (indexOp.equalsIgnoreCase("upsert") || indexOp.equalsIgnoreCase("update")) {
            sb.append("{\"update\": { \"_index\": \"");
            sb.append(index);
            sb.append("\", \"_type\": \"");
            sb.append(docType);
            sb.append("\", \"_id\": \"");
            sb.append(id);
            sb.append("\" }\n");
            sb.append("{\"doc\": ");
            sb.append(json);
            sb.append(", \"doc_as_upsert\": ");
            sb.append(indexOp.equalsIgnoreCase("upsert"));
            sb.append(" }\n");
        } else if (indexOp.equalsIgnoreCase("delete")) {
            sb.append("{\"delete\": { \"_index\": \"");
            sb.append(index);
            sb.append("\", \"_type\": \"");
            sb.append(docType);
            sb.append("\", \"_id\": \"");
            sb.append(id);
            sb.append("\" }\n");
        }
    }
    if (!flowFilesToTransfer.isEmpty()) {
        RequestBody requestBody = RequestBody.create(MediaType.parse("application/json"), sb.toString());
        final Response getResponse;
        try {
            getResponse = sendRequestToElasticsearch(okHttpClient, url, username, password, "PUT", requestBody);
        } catch (final Exception e) {
            logger.error("Routing to {} due to exception: {}", new Object[] { REL_FAILURE.getName(), e }, e);
            flowFilesToTransfer.forEach((flowFileToTransfer) -> {
                flowFileToTransfer = session.penalize(flowFileToTransfer);
                session.transfer(flowFileToTransfer, REL_FAILURE);
            });
            flowFilesToTransfer.clear();
            return;
        }
        final int statusCode = getResponse.code();
        if (isSuccess(statusCode)) {
            ResponseBody responseBody = getResponse.body();
            try {
                final byte[] bodyBytes = responseBody.bytes();
                JsonNode responseJson = parseJsonResponse(new ByteArrayInputStream(bodyBytes));
                boolean errors = responseJson.get("errors").asBoolean(false);
                if (errors) {
                    ArrayNode itemNodeArray = (ArrayNode) responseJson.get("items");
                    if (itemNodeArray.size() > 0) {
                        // All items are returned whether they succeeded or failed, so iterate through the item array
                        // at the same time as the flow file list, moving each to success or failure accordingly,
                        // but only keep the first error for logging
                        String errorReason = null;
                        for (int i = itemNodeArray.size() - 1; i >= 0; i--) {
                            JsonNode itemNode = itemNodeArray.get(i);
                            if (flowFilesToTransfer.size() > i) {
                                FlowFile flowFile = flowFilesToTransfer.remove(i);
                                int status = itemNode.findPath("status").asInt();
                                if (!isSuccess(status)) {
                                    if (errorReason == null) {
                                        // Use "result" if it is present; this happens for status codes like 404 Not Found, which may not have an error/reason
                                        String reason = itemNode.findPath("//result").asText();
                                        if (StringUtils.isEmpty(reason)) {
                                            // If there was no result, we expect an error with a string description in the "reason" field
                                            reason = itemNode.findPath("//error/reason").asText();
                                        }
                                        errorReason = reason;
                                        logger.error("Failed to process {} due to {}, transferring to failure", new Object[] { flowFile, errorReason });
                                    }
                                    flowFile = session.penalize(flowFile);
                                    session.transfer(flowFile, REL_FAILURE);
                                } else {
                                    session.transfer(flowFile, REL_SUCCESS);
                                    // Record provenance event
                                    session.getProvenanceReporter().send(flowFile, url.toString());
                                }
                            }
                        }
                    }
                }
                // Transfer any remaining flowfiles to success
                flowFilesToTransfer.forEach(file -> {
                    session.transfer(file, REL_SUCCESS);
                    // Record provenance event
                    session.getProvenanceReporter().send(file, url.toString());
                });
            } catch (IOException ioe) {
                // Something went wrong when parsing the response, log the error and route to failure
                logger.error("Error parsing Bulk API response: {}", new Object[] { ioe.getMessage() }, ioe);
                session.transfer(flowFilesToTransfer, REL_FAILURE);
                context.yield();
            }
        } else if (statusCode / 100 == 5) {
            // 5xx -> RETRY, but a server error might last a while, so yield
            logger.warn("Elasticsearch returned code {} with message {}, transferring flow file to retry. This is likely a server problem, yielding...", new Object[] { statusCode, getResponse.message() });
            session.transfer(flowFilesToTransfer, REL_RETRY);
            context.yield();
        } else {
            // 1xx, 3xx, 4xx, etc. -> NO RETRY
            logger.warn("Elasticsearch returned code {} with message {}, transferring flow file to failure", new Object[] { statusCode, getResponse.message() });
            session.transfer(flowFilesToTransfer, REL_FAILURE);
        }
        getResponse.close();
    }
}

Example 85 with ProcessSession

use of org.apache.nifi.processor.ProcessSession in project nifi by apache.

the class QueryElasticsearchHttp method getPage.

private int getPage(final Response getResponse, final URL url, final ProcessContext context, final ProcessSession session, FlowFile flowFile, final ComponentLog logger, final long startNanos, boolean targetIsContent) throws IOException {
    List<FlowFile> page = new ArrayList<>();
    final int statusCode = getResponse.code();
    if (isSuccess(statusCode)) {
        ResponseBody body = getResponse.body();
        final byte[] bodyBytes = body.bytes();
        JsonNode responseJson = parseJsonResponse(new ByteArrayInputStream(bodyBytes));
        JsonNode hits = responseJson.get("hits").get("hits");
        for (int i = 0; i < hits.size(); i++) {
            JsonNode hit = hits.get(i);
            String retrievedId = hit.get("_id").asText();
            String retrievedIndex = hit.get("_index").asText();
            String retrievedType = hit.get("_type").asText();
            FlowFile documentFlowFile = null;
            if (flowFile != null) {
                documentFlowFile = targetIsContent ? session.create(flowFile) : session.clone(flowFile);
            } else {
                documentFlowFile = session.create();
            }
            JsonNode source = hit.get("_source");
            documentFlowFile = session.putAttribute(documentFlowFile, "es.id", retrievedId);
            documentFlowFile = session.putAttribute(documentFlowFile, "es.index", retrievedIndex);
            documentFlowFile = session.putAttribute(documentFlowFile, "es.type", retrievedType);
            if (targetIsContent) {
                documentFlowFile = session.putAttribute(documentFlowFile, "filename", retrievedId);
                documentFlowFile = session.putAttribute(documentFlowFile, "mime.type", "application/json");
                documentFlowFile = session.write(documentFlowFile, out -> {
                    out.write(source.toString().getBytes());
                });
            } else {
                Map<String, String> attributes = new HashMap<>();
                for (Iterator<Entry<String, JsonNode>> it = source.fields(); it.hasNext(); ) {
                    Entry<String, JsonNode> entry = it.next();
                    attributes.put(ATTRIBUTE_PREFIX + entry.getKey(), entry.getValue().asText());
                }
                documentFlowFile = session.putAllAttributes(documentFlowFile, attributes);
            }
            page.add(documentFlowFile);
        }
        logger.debug("Elasticsearch retrieved " + responseJson.size() + " documents, routing to success");
        session.transfer(page, REL_SUCCESS);
    } else {
        try {
            // 5xx -> RETRY, but a server error might last a while, so yield
            if (statusCode / 100 == 5) {
                throw new RetryableException(String.format("Elasticsearch returned code %s with message %s, transferring flow file to retry. This is likely a server problem, yielding...", statusCode, getResponse.message()));
            } else if (context.hasIncomingConnection()) {
                // 1xx, 3xx, 4xx -> NO RETRY
                throw new UnretryableException(String.format("Elasticsearch returned code %s with message %s, transferring flow file to failure", statusCode, getResponse.message()));
            } else {
                logger.warn("Elasticsearch returned code {} with message {}", new Object[] { statusCode, getResponse.message() });
            }
        } finally {
            if (!page.isEmpty()) {
                session.remove(page);
                page.clear();
            }
        }
    }
    // emit provenance event
    final long millis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startNanos);
    if (!page.isEmpty()) {
        if (context.hasNonLoopConnection()) {
            page.forEach(f -> session.getProvenanceReporter().fetch(f, url.toExternalForm(), millis));
        } else {
            page.forEach(f -> session.getProvenanceReporter().receive(f, url.toExternalForm(), millis));
        }
    }
    return page.size();
}