Examples with ComponentLog org.apache.nifi.logging.ComponentLog used on opensource projects

Example 11 with ComponentLog

use of org.apache.nifi.logging.ComponentLog in project nifi by apache.

the class AbstractElasticsearchHttpProcessor method sendRequestToElasticsearch.

protected Response sendRequestToElasticsearch(OkHttpClient client, URL url, String username, String password, String verb, RequestBody body) throws IOException {
    final ComponentLog log = getLogger();
    Request.Builder requestBuilder = new Request.Builder().url(url);
    if ("get".equalsIgnoreCase(verb)) {
        requestBuilder = requestBuilder.get();
    } else if ("put".equalsIgnoreCase(verb)) {
        requestBuilder = requestBuilder.put(body);
    } else {
        throw new IllegalArgumentException("Elasticsearch REST API verb not supported by this processor: " + verb);
    }
    if (!StringUtils.isEmpty(username) && !StringUtils.isEmpty(password)) {
        String credential = Credentials.basic(username, password);
        requestBuilder = requestBuilder.header("Authorization", credential);
    }
    Request httpRequest = requestBuilder.build();
    log.debug("Sending Elasticsearch request to {}", new Object[] { url });
    Response responseHttp = client.newCall(httpRequest).execute();
    // store the status code and message
    int statusCode = responseHttp.code();
    if (statusCode == 0) {
        throw new IllegalStateException("Status code unknown, connection hasn't been attempted.");
    }
    log.debug("Received response from Elasticsearch with status code {}", new Object[] { statusCode });
    return responseHttp;
}

Example 12 with ComponentLog

use of org.apache.nifi.logging.ComponentLog in project nifi by apache.

the class AbstractElasticsearchTransportClientProcessor method createElasticsearchClient.

/**
 * Instantiate ElasticSearch Client. This should be called by subclasses' @OnScheduled method to create a client
 * if one does not yet exist. If called when scheduled, closeClient() should be called by the subclasses' @OnStopped
 * method so the client will be destroyed when the processor is stopped.
 *
 * @param context The context for this processor
 * @throws ProcessException if an error occurs while creating an Elasticsearch client
 */
@Override
protected void createElasticsearchClient(ProcessContext context) throws ProcessException {
    ComponentLog log = getLogger();
    if (esClient.get() != null) {
        return;
    }
    log.debug("Creating ElasticSearch Client");
    try {
        final String clusterName = context.getProperty(CLUSTER_NAME).evaluateAttributeExpressions().getValue();
        final String pingTimeout = context.getProperty(PING_TIMEOUT).evaluateAttributeExpressions().getValue();
        final String samplerInterval = context.getProperty(SAMPLER_INTERVAL).evaluateAttributeExpressions().getValue();
        final String username = context.getProperty(USERNAME).evaluateAttributeExpressions().getValue();
        final String password = context.getProperty(PASSWORD).evaluateAttributeExpressions().getValue();
        final SSLContextService sslService = context.getProperty(PROP_SSL_CONTEXT_SERVICE).asControllerService(SSLContextService.class);
        Settings.Builder settingsBuilder = Settings.settingsBuilder().put("cluster.name", clusterName).put("client.transport.ping_timeout", pingTimeout).put("client.transport.nodes_sampler_interval", samplerInterval);
        String shieldUrl = context.getProperty(PROP_SHIELD_LOCATION).evaluateAttributeExpressions().getValue();
        if (sslService != null) {
            settingsBuilder.put("shield.transport.ssl", "true").put("shield.ssl.keystore.path", sslService.getKeyStoreFile()).put("shield.ssl.keystore.password", sslService.getKeyStorePassword()).put("shield.ssl.truststore.path", sslService.getTrustStoreFile()).put("shield.ssl.truststore.password", sslService.getTrustStorePassword());
        }
        // Set username and password for Shield
        if (!StringUtils.isEmpty(username)) {
            StringBuffer shieldUser = new StringBuffer(username);
            if (!StringUtils.isEmpty(password)) {
                shieldUser.append(":");
                shieldUser.append(password);
            }
            settingsBuilder.put("shield.user", shieldUser);
        }
        TransportClient transportClient = getTransportClient(settingsBuilder, shieldUrl, username, password);
        final String hosts = context.getProperty(HOSTS).evaluateAttributeExpressions().getValue();
        esHosts = getEsHosts(hosts);
        if (esHosts != null) {
            for (final InetSocketAddress host : esHosts) {
                try {
                    transportClient.addTransportAddress(new InetSocketTransportAddress(host));
                } catch (IllegalArgumentException iae) {
                    log.error("Could not add transport address {}", new Object[] { host });
                }
            }
        }
        esClient.set(transportClient);
    } catch (Exception e) {
        log.error("Failed to create Elasticsearch client due to {}", new Object[] { e }, e);
        throw new ProcessException(e);
    }
}

Example 13 with ComponentLog

use of org.apache.nifi.logging.ComponentLog in project nifi by apache.

the class FetchElasticsearch method onTrigger.

@Override
public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException {
    FlowFile flowFile = session.get();
    if (flowFile == null) {
        return;
    }
    final String index = context.getProperty(INDEX).evaluateAttributeExpressions(flowFile).getValue();
    final String docId = context.getProperty(DOC_ID).evaluateAttributeExpressions(flowFile).getValue();
    final String docType = context.getProperty(TYPE).evaluateAttributeExpressions(flowFile).getValue();
    final Charset charset = Charset.forName(context.getProperty(CHARSET).evaluateAttributeExpressions(flowFile).getValue());
    final ComponentLog logger = getLogger();
    try {
        logger.debug("Fetching {}/{}/{} from Elasticsearch", new Object[] { index, docType, docId });
        final long startNanos = System.nanoTime();
        GetRequestBuilder getRequestBuilder = esClient.get().prepareGet(index, docType, docId);
        if (authToken != null) {
            getRequestBuilder.putHeader("Authorization", authToken);
        }
        final GetResponse getResponse = getRequestBuilder.execute().actionGet();
        if (getResponse == null || !getResponse.isExists()) {
            logger.debug("Failed to read {}/{}/{} from Elasticsearch: Document not found", new Object[] { index, docType, docId });
            // We couldn't find the document, so penalize it and send it to "not found"
            flowFile = session.penalize(flowFile);
            session.transfer(flowFile, REL_NOT_FOUND);
        } else {
            flowFile = session.putAttribute(flowFile, "filename", docId);
            flowFile = session.putAttribute(flowFile, "es.index", index);
            flowFile = session.putAttribute(flowFile, "es.type", docType);
            flowFile = session.write(flowFile, new OutputStreamCallback() {

                @Override
                public void process(OutputStream out) throws IOException {
                    out.write(getResponse.getSourceAsString().getBytes(charset));
                }
            });
            logger.debug("Elasticsearch document " + docId + " fetched, routing to success");
            final long millis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startNanos);
            final String uri = context.getProperty(HOSTS).evaluateAttributeExpressions().getValue() + "/" + index + "/" + docType + "/" + docId;
            session.getProvenanceReporter().fetch(flowFile, uri, millis);
            session.transfer(flowFile, REL_SUCCESS);
        }
    } catch (NoNodeAvailableException | ElasticsearchTimeoutException | ReceiveTimeoutTransportException | NodeClosedException exceptionToRetry) {
        logger.error("Failed to read into Elasticsearch due to {}, this may indicate an error in configuration " + "(hosts, username/password, etc.). Routing to retry", new Object[] { exceptionToRetry.getLocalizedMessage() }, exceptionToRetry);
        session.transfer(flowFile, REL_RETRY);
        context.yield();
    } catch (Exception e) {
        logger.error("Failed to read {} from Elasticsearch due to {}", new Object[] { flowFile, e.getLocalizedMessage() }, e);
        session.transfer(flowFile, REL_FAILURE);
        context.yield();
    }
}

Example 14 with ComponentLog

use of org.apache.nifi.logging.ComponentLog in project nifi by apache.

the class FetchElasticsearchHttp method onTrigger.

@Override
public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException {
    FlowFile flowFile = null;
    if (context.hasIncomingConnection()) {
        flowFile = session.get();
        // we know that we should run only if we have a FlowFile.
        if (flowFile == null && context.hasNonLoopConnection()) {
            return;
        }
    }
    OkHttpClient okHttpClient = getClient();
    if (flowFile == null) {
        flowFile = session.create();
    }
    final String index = context.getProperty(INDEX).evaluateAttributeExpressions(flowFile).getValue();
    final String docId = context.getProperty(DOC_ID).evaluateAttributeExpressions(flowFile).getValue();
    final String docType = context.getProperty(TYPE).evaluateAttributeExpressions(flowFile).getValue();
    final String fields = context.getProperty(FIELDS).isSet() ? context.getProperty(FIELDS).evaluateAttributeExpressions(flowFile).getValue() : null;
    // Authentication
    final String username = context.getProperty(USERNAME).evaluateAttributeExpressions(flowFile).getValue();
    final String password = context.getProperty(PASSWORD).evaluateAttributeExpressions().getValue();
    final ComponentLog logger = getLogger();
    Response getResponse = null;
    try {
        logger.debug("Fetching {}/{}/{} from Elasticsearch", new Object[] { index, docType, docId });
        // read the url property from the context
        final String urlstr = StringUtils.trimToEmpty(context.getProperty(ES_URL).evaluateAttributeExpressions().getValue());
        final URL url = buildRequestURL(urlstr, docId, index, docType, fields, context);
        final long startNanos = System.nanoTime();
        getResponse = sendRequestToElasticsearch(okHttpClient, url, username, password, "GET", null);
        final int statusCode = getResponse.code();
        if (isSuccess(statusCode)) {
            ResponseBody body = getResponse.body();
            final byte[] bodyBytes = body.bytes();
            JsonNode responseJson = parseJsonResponse(new ByteArrayInputStream(bodyBytes));
            boolean found = responseJson.get("found").asBoolean(false);
            String retrievedIndex = responseJson.get("_index").asText();
            String retrievedType = responseJson.get("_type").asText();
            String retrievedId = responseJson.get("_id").asText();
            if (found) {
                JsonNode source = responseJson.get("_source");
                flowFile = session.putAttribute(flowFile, "filename", retrievedId);
                flowFile = session.putAttribute(flowFile, "es.index", retrievedIndex);
                flowFile = session.putAttribute(flowFile, "es.type", retrievedType);
                if (source != null) {
                    flowFile = session.write(flowFile, out -> {
                        out.write(source.toString().getBytes());
                    });
                }
                logger.debug("Elasticsearch document " + retrievedId + " fetched, routing to success");
                // emit provenance event
                final long millis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startNanos);
                if (context.hasNonLoopConnection()) {
                    session.getProvenanceReporter().fetch(flowFile, url.toExternalForm(), millis);
                } else {
                    session.getProvenanceReporter().receive(flowFile, url.toExternalForm(), millis);
                }
                session.transfer(flowFile, REL_SUCCESS);
            } else {
                logger.debug("Failed to read {}/{}/{} from Elasticsearch: Document not found", new Object[] { index, docType, docId });
                // We couldn't find the document, so send it to "not found"
                session.transfer(flowFile, REL_NOT_FOUND);
            }
        } else {
            if (statusCode == 404) {
                logger.warn("Failed to read {}/{}/{} from Elasticsearch: Document not found", new Object[] { index, docType, docId });
                // We couldn't find the document, so penalize it and send it to "not found"
                session.transfer(flowFile, REL_NOT_FOUND);
            } else {
                // 5xx -> RETRY, but a server error might last a while, so yield
                if (statusCode / 100 == 5) {
                    logger.warn("Elasticsearch returned code {} with message {}, transferring flow file to retry. This is likely a server problem, yielding...", new Object[] { statusCode, getResponse.message() });
                    session.transfer(flowFile, REL_RETRY);
                    context.yield();
                } else if (context.hasIncomingConnection()) {
                    // 1xx, 3xx, 4xx -> NO RETRY
                    logger.warn("Elasticsearch returned code {} with message {}, transferring flow file to failure", new Object[] { statusCode, getResponse.message() });
                    session.transfer(flowFile, REL_FAILURE);
                } else {
                    logger.warn("Elasticsearch returned code {} with message {}", new Object[] { statusCode, getResponse.message() });
                    session.remove(flowFile);
                }
            }
        }
    } catch (IOException ioe) {
        logger.error("Failed to read from Elasticsearch due to {}, this may indicate an error in configuration " + "(hosts, username/password, etc.). Routing to retry", new Object[] { ioe.getLocalizedMessage() }, ioe);
        if (context.hasIncomingConnection()) {
            session.transfer(flowFile, REL_RETRY);
        } else {
            session.remove(flowFile);
        }
        context.yield();
    } catch (Exception e) {
        logger.error("Failed to read {} from Elasticsearch due to {}", new Object[] { flowFile, e.getLocalizedMessage() }, e);
        if (context.hasIncomingConnection()) {
            session.transfer(flowFile, REL_FAILURE);
        } else {
            session.remove(flowFile);
        }
        context.yield();
    } finally {
        if (getResponse != null) {
            getResponse.close();
        }
    }
}

Example 15 with ComponentLog

use of org.apache.nifi.logging.ComponentLog in project nifi by apache.

the class PutElasticsearch method onTrigger.

@Override
public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException {
    final ComponentLog logger = getLogger();
    final String id_attribute = context.getProperty(ID_ATTRIBUTE).getValue();
    final int batchSize = context.getProperty(BATCH_SIZE).evaluateAttributeExpressions().asInteger();
    final List<FlowFile> flowFiles = session.get(batchSize);
    if (flowFiles.isEmpty()) {
        return;
    }
    // Keep track of the list of flow files that need to be transferred. As they are transferred, remove them from the list.
    List<FlowFile> flowFilesToTransfer = new LinkedList<>(flowFiles);
    try {
        final BulkRequestBuilder bulk = esClient.get().prepareBulk();
        if (authToken != null) {
            bulk.putHeader("Authorization", authToken);
        }
        for (FlowFile file : flowFiles) {
            final String index = context.getProperty(INDEX).evaluateAttributeExpressions(file).getValue();
            final String docType = context.getProperty(TYPE).evaluateAttributeExpressions(file).getValue();
            final String indexOp = context.getProperty(INDEX_OP).evaluateAttributeExpressions(file).getValue();
            final Charset charset = Charset.forName(context.getProperty(CHARSET).evaluateAttributeExpressions(file).getValue());
            final String id = file.getAttribute(id_attribute);
            if (id == null) {
                logger.error("No value in identifier attribute {} for {}, transferring to failure", new Object[] { id_attribute, file });
                flowFilesToTransfer.remove(file);
                session.transfer(file, REL_FAILURE);
            } else {
                session.read(file, new InputStreamCallback() {

                    @Override
                    public void process(final InputStream in) throws IOException {
                        String json = IOUtils.toString(in, charset).replace("\r\n", " ").replace('\n', ' ').replace('\r', ' ');
                        if (indexOp.equalsIgnoreCase("index")) {
                            bulk.add(esClient.get().prepareIndex(index, docType, id).setSource(json.getBytes(charset)));
                        } else if (indexOp.equalsIgnoreCase("upsert")) {
                            bulk.add(esClient.get().prepareUpdate(index, docType, id).setDoc(json.getBytes(charset)).setDocAsUpsert(true));
                        } else if (indexOp.equalsIgnoreCase("update")) {
                            bulk.add(esClient.get().prepareUpdate(index, docType, id).setDoc(json.getBytes(charset)));
                        } else {
                            throw new IOException("Index operation: " + indexOp + " not supported.");
                        }
                    }
                });
            }
        }
        final BulkResponse response = bulk.execute().actionGet();
        if (response.hasFailures()) {
            // Responses are guaranteed to be in order, remove them in reverse order
            BulkItemResponse[] responses = response.getItems();
            if (responses != null && responses.length > 0) {
                for (int i = responses.length - 1; i >= 0; i--) {
                    final FlowFile flowFile = flowFilesToTransfer.get(i);
                    if (responses[i].isFailed()) {
                        logger.error("Failed to insert {} into Elasticsearch due to {}, transferring to failure", new Object[] { flowFile, responses[i].getFailure().getMessage() });
                        session.transfer(flowFile, REL_FAILURE);
                    } else {
                        session.getProvenanceReporter().send(flowFile, context.getProperty(HOSTS).evaluateAttributeExpressions().getValue() + "/" + responses[i].getIndex());
                        session.transfer(flowFile, REL_SUCCESS);
                    }
                    flowFilesToTransfer.remove(flowFile);
                }
            }
        }
        // Transfer any remaining flowfiles to success
        flowFilesToTransfer.forEach(file -> {
            session.transfer(file, REL_SUCCESS);
            // Record provenance event
            session.getProvenanceReporter().send(file, context.getProperty(HOSTS).evaluateAttributeExpressions().getValue() + "/" + context.getProperty(INDEX).evaluateAttributeExpressions(file).getValue());
        });
    } catch (NoNodeAvailableException | ElasticsearchTimeoutException | ReceiveTimeoutTransportException | NodeClosedException exceptionToRetry) {
        // Authorization errors and other problems are often returned as NoNodeAvailableExceptions without a
        // traceable cause. However the cause seems to be logged, just not available to this caught exception.
        // Since the error message will show up as a bulletin, we make specific mention to check the logs for
        // more details.
        logger.error("Failed to insert into Elasticsearch due to {}. More detailed information may be available in " + "the NiFi logs.", new Object[] { exceptionToRetry.getLocalizedMessage() }, exceptionToRetry);
        session.transfer(flowFilesToTransfer, REL_RETRY);
        context.yield();
    } catch (Exception exceptionToFail) {
        logger.error("Failed to insert into Elasticsearch due to {}, transferring to failure", new Object[] { exceptionToFail.getLocalizedMessage() }, exceptionToFail);
        session.transfer(flowFilesToTransfer, REL_FAILURE);
        context.yield();
    }
}