Examples with OkHttpClient okhttp3.OkHttpClient used on opensource projects

Example 36 with OkHttpClient

use of okhttp3.OkHttpClient in project nifi by apache.

the class AbstractElasticsearchHttpProcessor method sendRequestToElasticsearch.

protected Response sendRequestToElasticsearch(OkHttpClient client, URL url, String username, String password, String verb, RequestBody body) throws IOException {
    final ComponentLog log = getLogger();
    Request.Builder requestBuilder = new Request.Builder().url(url);
    if ("get".equalsIgnoreCase(verb)) {
        requestBuilder = requestBuilder.get();
    } else if ("put".equalsIgnoreCase(verb)) {
        requestBuilder = requestBuilder.put(body);
    } else {
        throw new IllegalArgumentException("Elasticsearch REST API verb not supported by this processor: " + verb);
    }
    if (!StringUtils.isEmpty(username) && !StringUtils.isEmpty(password)) {
        String credential = Credentials.basic(username, password);
        requestBuilder = requestBuilder.header("Authorization", credential);
    }
    Request httpRequest = requestBuilder.build();
    log.debug("Sending Elasticsearch request to {}", new Object[] { url });
    Response responseHttp = client.newCall(httpRequest).execute();
    // store the status code and message
    int statusCode = responseHttp.code();
    if (statusCode == 0) {
        throw new IllegalStateException("Status code unknown, connection hasn't been attempted.");
    }
    log.debug("Received response from Elasticsearch with status code {}", new Object[] { statusCode });
    return responseHttp;
}

Example 37 with OkHttpClient

use of okhttp3.OkHttpClient in project nifi by apache.

the class FetchElasticsearchHttp method onTrigger.

@Override
public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException {
    FlowFile flowFile = null;
    if (context.hasIncomingConnection()) {
        flowFile = session.get();
        // we know that we should run only if we have a FlowFile.
        if (flowFile == null && context.hasNonLoopConnection()) {
            return;
        }
    }
    OkHttpClient okHttpClient = getClient();
    if (flowFile == null) {
        flowFile = session.create();
    }
    final String index = context.getProperty(INDEX).evaluateAttributeExpressions(flowFile).getValue();
    final String docId = context.getProperty(DOC_ID).evaluateAttributeExpressions(flowFile).getValue();
    final String docType = context.getProperty(TYPE).evaluateAttributeExpressions(flowFile).getValue();
    final String fields = context.getProperty(FIELDS).isSet() ? context.getProperty(FIELDS).evaluateAttributeExpressions(flowFile).getValue() : null;
    // Authentication
    final String username = context.getProperty(USERNAME).evaluateAttributeExpressions(flowFile).getValue();
    final String password = context.getProperty(PASSWORD).evaluateAttributeExpressions().getValue();
    final ComponentLog logger = getLogger();
    Response getResponse = null;
    try {
        logger.debug("Fetching {}/{}/{} from Elasticsearch", new Object[] { index, docType, docId });
        // read the url property from the context
        final String urlstr = StringUtils.trimToEmpty(context.getProperty(ES_URL).evaluateAttributeExpressions().getValue());
        final URL url = buildRequestURL(urlstr, docId, index, docType, fields, context);
        final long startNanos = System.nanoTime();
        getResponse = sendRequestToElasticsearch(okHttpClient, url, username, password, "GET", null);
        final int statusCode = getResponse.code();
        if (isSuccess(statusCode)) {
            ResponseBody body = getResponse.body();
            final byte[] bodyBytes = body.bytes();
            JsonNode responseJson = parseJsonResponse(new ByteArrayInputStream(bodyBytes));
            boolean found = responseJson.get("found").asBoolean(false);
            String retrievedIndex = responseJson.get("_index").asText();
            String retrievedType = responseJson.get("_type").asText();
            String retrievedId = responseJson.get("_id").asText();
            if (found) {
                JsonNode source = responseJson.get("_source");
                flowFile = session.putAttribute(flowFile, "filename", retrievedId);
                flowFile = session.putAttribute(flowFile, "es.index", retrievedIndex);
                flowFile = session.putAttribute(flowFile, "es.type", retrievedType);
                if (source != null) {
                    flowFile = session.write(flowFile, out -> {
                        out.write(source.toString().getBytes());
                    });
                }
                logger.debug("Elasticsearch document " + retrievedId + " fetched, routing to success");
                // emit provenance event
                final long millis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startNanos);
                if (context.hasNonLoopConnection()) {
                    session.getProvenanceReporter().fetch(flowFile, url.toExternalForm(), millis);
                } else {
                    session.getProvenanceReporter().receive(flowFile, url.toExternalForm(), millis);
                }
                session.transfer(flowFile, REL_SUCCESS);
            } else {
                logger.debug("Failed to read {}/{}/{} from Elasticsearch: Document not found", new Object[] { index, docType, docId });
                // We couldn't find the document, so send it to "not found"
                session.transfer(flowFile, REL_NOT_FOUND);
            }
        } else {
            if (statusCode == 404) {
                logger.warn("Failed to read {}/{}/{} from Elasticsearch: Document not found", new Object[] { index, docType, docId });
                // We couldn't find the document, so penalize it and send it to "not found"
                session.transfer(flowFile, REL_NOT_FOUND);
            } else {
                // 5xx -> RETRY, but a server error might last a while, so yield
                if (statusCode / 100 == 5) {
                    logger.warn("Elasticsearch returned code {} with message {}, transferring flow file to retry. This is likely a server problem, yielding...", new Object[] { statusCode, getResponse.message() });
                    session.transfer(flowFile, REL_RETRY);
                    context.yield();
                } else if (context.hasIncomingConnection()) {
                    // 1xx, 3xx, 4xx -> NO RETRY
                    logger.warn("Elasticsearch returned code {} with message {}, transferring flow file to failure", new Object[] { statusCode, getResponse.message() });
                    session.transfer(flowFile, REL_FAILURE);
                } else {
                    logger.warn("Elasticsearch returned code {} with message {}", new Object[] { statusCode, getResponse.message() });
                    session.remove(flowFile);
                }
            }
        }
    } catch (IOException ioe) {
        logger.error("Failed to read from Elasticsearch due to {}, this may indicate an error in configuration " + "(hosts, username/password, etc.). Routing to retry", new Object[] { ioe.getLocalizedMessage() }, ioe);
        if (context.hasIncomingConnection()) {
            session.transfer(flowFile, REL_RETRY);
        } else {
            session.remove(flowFile);
        }
        context.yield();
    } catch (Exception e) {
        logger.error("Failed to read {} from Elasticsearch due to {}", new Object[] { flowFile, e.getLocalizedMessage() }, e);
        if (context.hasIncomingConnection()) {
            session.transfer(flowFile, REL_FAILURE);
        } else {
            session.remove(flowFile);
        }
        context.yield();
    } finally {
        if (getResponse != null) {
            getResponse.close();
        }
    }
}

Example 38 with OkHttpClient

use of okhttp3.OkHttpClient in project nifi by apache.

the class ScrollElasticsearchHttp method onTrigger.

@Override
public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException {
    try {
        if (isQueryFinished(context.getStateManager())) {
            getLogger().trace("Query has been marked finished in the state manager.  " + "To run another query, clear the state.");
            return;
        }
    } catch (IOException e) {
        throw new ProcessException("Could not retrieve state", e);
    }
    OkHttpClient okHttpClient = getClient();
    FlowFile flowFile = session.create();
    final String index = context.getProperty(INDEX).evaluateAttributeExpressions(flowFile).getValue();
    final String query = context.getProperty(QUERY).evaluateAttributeExpressions(flowFile).getValue();
    final String docType = context.getProperty(TYPE).evaluateAttributeExpressions(flowFile).getValue();
    final int pageSize = context.getProperty(PAGE_SIZE).evaluateAttributeExpressions(flowFile).asInteger().intValue();
    final String fields = context.getProperty(FIELDS).isSet() ? context.getProperty(FIELDS).evaluateAttributeExpressions(flowFile).getValue() : null;
    final String sort = context.getProperty(SORT).isSet() ? context.getProperty(SORT).evaluateAttributeExpressions(flowFile).getValue() : null;
    final String scroll = context.getProperty(SCROLL_DURATION).isSet() ? context.getProperty(SCROLL_DURATION).evaluateAttributeExpressions(flowFile).getValue() : null;
    // Authentication
    final String username = context.getProperty(USERNAME).evaluateAttributeExpressions().getValue();
    final String password = context.getProperty(PASSWORD).evaluateAttributeExpressions().getValue();
    final ComponentLog logger = getLogger();
    try {
        String scrollId = loadScrollId(context.getStateManager());
        // read the url property from the context
        final String urlstr = StringUtils.trimToEmpty(context.getProperty(ES_URL).evaluateAttributeExpressions().getValue());
        if (scrollId != null) {
            final URL scrollurl = buildRequestURL(urlstr, query, index, docType, fields, sort, scrollId, pageSize, scroll, context);
            final long startNanos = System.nanoTime();
            final Response getResponse = sendRequestToElasticsearch(okHttpClient, scrollurl, username, password, "GET", null);
            this.getPage(getResponse, scrollurl, context, session, flowFile, logger, startNanos);
            getResponse.close();
        } else {
            logger.debug("Querying {}/{} from Elasticsearch: {}", new Object[] { index, docType, query });
            // read the url property from the context
            final URL queryUrl = buildRequestURL(urlstr, query, index, docType, fields, sort, scrollId, pageSize, scroll, context);
            final long startNanos = System.nanoTime();
            final Response getResponse = sendRequestToElasticsearch(okHttpClient, queryUrl, username, password, "GET", null);
            this.getPage(getResponse, queryUrl, context, session, flowFile, logger, startNanos);
            getResponse.close();
        }
    } catch (IOException ioe) {
        logger.error("Failed to read from Elasticsearch due to {}, this may indicate an error in configuration " + "(hosts, username/password, etc.).", new Object[] { ioe.getLocalizedMessage() }, ioe);
        session.remove(flowFile);
        context.yield();
    } catch (Exception e) {
        logger.error("Failed to read {} from Elasticsearch due to {}", new Object[] { flowFile, e.getLocalizedMessage() }, e);
        session.transfer(flowFile, REL_FAILURE);
        context.yield();
    }
}

Example 39 with OkHttpClient

use of okhttp3.OkHttpClient in project nifi by apache.

the class InvokeHTTP method onTrigger.

@Override
public void onTrigger(ProcessContext context, ProcessSession session) throws ProcessException {
    OkHttpClient okHttpClient = okHttpClientAtomicReference.get();
    FlowFile requestFlowFile = session.get();
    // Checking to see if the property to put the body of the response in an attribute was set
    boolean putToAttribute = context.getProperty(PROP_PUT_OUTPUT_IN_ATTRIBUTE).isSet();
    if (requestFlowFile == null) {
        if (context.hasNonLoopConnection()) {
            return;
        }
        String request = context.getProperty(PROP_METHOD).evaluateAttributeExpressions().getValue().toUpperCase();
        if ("POST".equals(request) || "PUT".equals(request) || "PATCH".equals(request)) {
            return;
        } else if (putToAttribute) {
            requestFlowFile = session.create();
        }
    }
    // Setting some initial variables
    final int maxAttributeSize = context.getProperty(PROP_PUT_ATTRIBUTE_MAX_LENGTH).asInteger();
    final ComponentLog logger = getLogger();
    // log ETag cache metrics
    final boolean eTagEnabled = context.getProperty(PROP_USE_ETAG).asBoolean();
    if (eTagEnabled && logger.isDebugEnabled()) {
        final Cache cache = okHttpClient.cache();
        logger.debug("OkHttp ETag cache metrics :: Request Count: {} | Network Count: {} | Hit Count: {}", new Object[] { cache.requestCount(), cache.networkCount(), cache.hitCount() });
    }
    // Every request/response cycle has a unique transaction id which will be stored as a flowfile attribute.
    final UUID txId = UUID.randomUUID();
    FlowFile responseFlowFile = null;
    try {
        // read the url property from the context
        final String urlstr = trimToEmpty(context.getProperty(PROP_URL).evaluateAttributeExpressions(requestFlowFile).getValue());
        final URL url = new URL(urlstr);
        Request httpRequest = configureRequest(context, session, requestFlowFile, url);
        // log request
        logRequest(logger, httpRequest);
        // emit send provenance event if successfully sent to the server
        if (httpRequest.body() != null) {
            session.getProvenanceReporter().send(requestFlowFile, url.toExternalForm(), true);
        }
        final long startNanos = System.nanoTime();
        Response responseHttp = okHttpClient.newCall(httpRequest).execute();
        // output the raw response headers (DEBUG level only)
        logResponse(logger, url, responseHttp);
        // store the status code and message
        int statusCode = responseHttp.code();
        String statusMessage = responseHttp.message();
        if (statusCode == 0) {
            throw new IllegalStateException("Status code unknown, connection hasn't been attempted.");
        }
        // Create a map of the status attributes that are always written to the request and response FlowFiles
        Map<String, String> statusAttributes = new HashMap<>();
        statusAttributes.put(STATUS_CODE, String.valueOf(statusCode));
        statusAttributes.put(STATUS_MESSAGE, statusMessage);
        statusAttributes.put(REQUEST_URL, url.toExternalForm());
        statusAttributes.put(TRANSACTION_ID, txId.toString());
        if (requestFlowFile != null) {
            requestFlowFile = session.putAllAttributes(requestFlowFile, statusAttributes);
        }
        // If the property to add the response headers to the request flowfile is true then add them
        if (context.getProperty(PROP_ADD_HEADERS_TO_REQUEST).asBoolean() && requestFlowFile != null) {
            // write the response headers as attributes
            // this will overwrite any existing flowfile attributes
            requestFlowFile = session.putAllAttributes(requestFlowFile, convertAttributesFromHeaders(url, responseHttp));
        }
        boolean outputBodyToRequestAttribute = (!isSuccess(statusCode) || putToAttribute) && requestFlowFile != null;
        boolean outputBodyToResponseContent = (isSuccess(statusCode) && !putToAttribute) || context.getProperty(PROP_OUTPUT_RESPONSE_REGARDLESS).asBoolean();
        ResponseBody responseBody = responseHttp.body();
        boolean bodyExists = responseBody != null;
        InputStream responseBodyStream = null;
        SoftLimitBoundedByteArrayOutputStream outputStreamToRequestAttribute = null;
        TeeInputStream teeInputStream = null;
        try {
            responseBodyStream = bodyExists ? responseBody.byteStream() : null;
            if (responseBodyStream != null && outputBodyToRequestAttribute && outputBodyToResponseContent) {
                outputStreamToRequestAttribute = new SoftLimitBoundedByteArrayOutputStream(maxAttributeSize);
                teeInputStream = new TeeInputStream(responseBodyStream, outputStreamToRequestAttribute);
            }
            if (outputBodyToResponseContent) {
                // clone the flowfile to capture the response
                if (requestFlowFile != null) {
                    responseFlowFile = session.create(requestFlowFile);
                } else {
                    responseFlowFile = session.create();
                }
                // write attributes to response flowfile
                responseFlowFile = session.putAllAttributes(responseFlowFile, statusAttributes);
                // write the response headers as attributes
                // this will overwrite any existing flowfile attributes
                responseFlowFile = session.putAllAttributes(responseFlowFile, convertAttributesFromHeaders(url, responseHttp));
                // can potentially be null in edge cases
                if (bodyExists) {
                    // write content type attribute to response flowfile if it is available
                    if (responseBody.contentType() != null) {
                        responseFlowFile = session.putAttribute(responseFlowFile, CoreAttributes.MIME_TYPE.key(), responseBody.contentType().toString());
                    }
                    if (teeInputStream != null) {
                        responseFlowFile = session.importFrom(teeInputStream, responseFlowFile);
                    } else {
                        responseFlowFile = session.importFrom(responseBodyStream, responseFlowFile);
                    }
                    // emit provenance event
                    final long millis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startNanos);
                    if (requestFlowFile != null) {
                        session.getProvenanceReporter().fetch(responseFlowFile, url.toExternalForm(), millis);
                    } else {
                        session.getProvenanceReporter().receive(responseFlowFile, url.toExternalForm(), millis);
                    }
                }
            }
            // if not successful and request flowfile is not null, store the response body into a flowfile attribute
            if (outputBodyToRequestAttribute && bodyExists) {
                String attributeKey = context.getProperty(PROP_PUT_OUTPUT_IN_ATTRIBUTE).evaluateAttributeExpressions(requestFlowFile).getValue();
                if (attributeKey == null) {
                    attributeKey = RESPONSE_BODY;
                }
                byte[] outputBuffer;
                int size;
                if (outputStreamToRequestAttribute != null) {
                    outputBuffer = outputStreamToRequestAttribute.getBuffer();
                    size = outputStreamToRequestAttribute.size();
                } else {
                    outputBuffer = new byte[maxAttributeSize];
                    size = StreamUtils.fillBuffer(responseBodyStream, outputBuffer, false);
                }
                String bodyString = new String(outputBuffer, 0, size, getCharsetFromMediaType(responseBody.contentType()));
                requestFlowFile = session.putAttribute(requestFlowFile, attributeKey, bodyString);
                final long millis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startNanos);
                session.getProvenanceReporter().modifyAttributes(requestFlowFile, "The " + attributeKey + " has been added. The value of which is the body of a http call to " + url.toExternalForm() + ". It took " + millis + "millis,");
            }
        } finally {
            if (outputStreamToRequestAttribute != null) {
                outputStreamToRequestAttribute.close();
                outputStreamToRequestAttribute = null;
            }
            if (teeInputStream != null) {
                teeInputStream.close();
                teeInputStream = null;
            } else if (responseBodyStream != null) {
                responseBodyStream.close();
                responseBodyStream = null;
            }
        }
        route(requestFlowFile, responseFlowFile, session, context, statusCode);
    } catch (final Exception e) {
        // penalize or yield
        if (requestFlowFile != null) {
            logger.error("Routing to {} due to exception: {}", new Object[] { REL_FAILURE.getName(), e }, e);
            requestFlowFile = session.penalize(requestFlowFile);
            requestFlowFile = session.putAttribute(requestFlowFile, EXCEPTION_CLASS, e.getClass().getName());
            requestFlowFile = session.putAttribute(requestFlowFile, EXCEPTION_MESSAGE, e.getMessage());
            // transfer original to failure
            session.transfer(requestFlowFile, REL_FAILURE);
        } else {
            logger.error("Yielding processor due to exception encountered as a source processor: {}", e);
            context.yield();
        }
        // cleanup response flowfile, if applicable
        try {
            if (responseFlowFile != null) {
                session.remove(responseFlowFile);
            }
        } catch (final Exception e1) {
            logger.error("Could not cleanup response flowfile due to exception: {}", new Object[] { e1 }, e1);
        }
    }
}

Example 40 with OkHttpClient

use of okhttp3.OkHttpClient in project couchbase-lite-android by couchbase.

the class ReplicatorWithSyncGatewayDBTest method remote_DELETE_db.

private boolean remote_DELETE_db(String db) throws IOException {
    OkHttpClient client = new OkHttpClient();
    String url = String.format(Locale.ENGLISH, "http://%s:4985/%s/", this.config.remoteHost(), db);
    okhttp3.Request request = new okhttp3.Request.Builder().url(url).delete().build();
    Response response = client.newCall(request).execute();
    return response.code() >= 200 && response.code() < 300;
}