Examples with RestHighLevelClient org.elasticsearch.client.RestHighLevelClient used on opensource projects

Example 6 with RestHighLevelClient

use of org.elasticsearch.client.RestHighLevelClient in project logging-log4j2 by apache.

the class LogstashIT method testEvents.

private static void testEvents(final List<LogEvent> logEvents) throws IOException {
    try (final RestHighLevelClient client = createClient()) {
        final Appender appender = createStartedAppender(JSON_TEMPLATE_GELF_LAYOUT, MavenHardcodedConstants.LS_GELF_INPUT_PORT);
        try {
            // Append events.
            LOGGER.info("appending events");
            logEvents.forEach(appender::append);
            LOGGER.info("completed appending events");
            // Wait all messages to arrive.
            Awaitility.await().atMost(Duration.ofSeconds(60)).pollDelay(Duration.ofSeconds(2)).until(() -> queryDocumentCount(client) == LOG_EVENT_COUNT);
            // Verify indexed messages.
            final Set<String> expectedMessages = logEvents.stream().map(LogstashIT::expectedLogstashMessageField).collect(Collectors.toSet());
            final Set<String> actualMessages = queryDocuments(client).stream().map(source -> (String) source.get(ES_INDEX_MESSAGE_FIELD_NAME)).filter(Objects::nonNull).collect(Collectors.toSet());
            Assertions.assertThat(actualMessages).isEqualTo(expectedMessages);
        } finally {
            appender.stop();
        }
    }
}

Example 7 with RestHighLevelClient

use of org.elasticsearch.client.RestHighLevelClient in project logging-log4j2 by apache.

the class LogstashIT method appendAndCollect.

private static <K> Map<K, Object> appendAndCollect(final List<LogEvent> logEvents, final Layout<?> layout, final int port, final Function<Map<String, Object>, K> keyMapper, final Set<String> excludedKeys) throws IOException {
    try (final RestHighLevelClient client = createClient()) {
        final Appender appender = createStartedAppender(layout, port);
        try {
            // Append the event.
            LOGGER.info("appending events");
            logEvents.forEach(appender::append);
            LOGGER.info("completed appending events");
            // Wait the message to arrive.
            Awaitility.await().atMost(Duration.ofSeconds(60)).pollDelay(Duration.ofSeconds(2)).until(() -> queryDocumentCount(client) == LOG_EVENT_COUNT);
            // Retrieve the persisted messages.
            return queryDocuments(client).stream().collect(Collectors.toMap(keyMapper, (final Map<String, Object> source) -> {
                excludedKeys.forEach(source::remove);
                return source;
            }));
        } finally {
            appender.stop();
        }
    }
}

Example 8 with RestHighLevelClient

use of org.elasticsearch.client.RestHighLevelClient in project flink by apache.

the class Elasticsearch7SinkBuilder method getBulkProcessorBuilderFactory.

@Override
protected BulkProcessorBuilderFactory getBulkProcessorBuilderFactory() {
    return new BulkProcessorBuilderFactory() {

        @Override
        public BulkProcessor.Builder apply(RestHighLevelClient client, BulkProcessorConfig bulkProcessorConfig, BulkProcessor.Listener listener) {
            BulkProcessor.Builder builder = BulkProcessor.builder(new // This cannot be inlined as a
            BulkRequestConsumerFactory() {

                // lambda because then
                // deserialization fails
                @Override
                public void accept(BulkRequest bulkRequest, ActionListener<BulkResponse> bulkResponseActionListener) {
                    client.bulkAsync(bulkRequest, RequestOptions.DEFAULT, bulkResponseActionListener);
                }
            }, listener);
            if (bulkProcessorConfig.getBulkFlushMaxActions() != -1) {
                builder.setBulkActions(bulkProcessorConfig.getBulkFlushMaxActions());
            }
            if (bulkProcessorConfig.getBulkFlushMaxMb() != -1) {
                builder.setBulkSize(new ByteSizeValue(bulkProcessorConfig.getBulkFlushMaxMb(), ByteSizeUnit.MB));
            }
            if (bulkProcessorConfig.getBulkFlushInterval() != -1) {
                builder.setFlushInterval(new TimeValue(bulkProcessorConfig.getBulkFlushInterval()));
            }
            BackoffPolicy backoffPolicy;
            final TimeValue backoffDelay = new TimeValue(bulkProcessorConfig.getBulkFlushBackOffDelay());
            final int maxRetryCount = bulkProcessorConfig.getBulkFlushBackoffRetries();
            switch(bulkProcessorConfig.getFlushBackoffType()) {
                case CONSTANT:
                    backoffPolicy = BackoffPolicy.constantBackoff(backoffDelay, maxRetryCount);
                    break;
                case EXPONENTIAL:
                    backoffPolicy = BackoffPolicy.exponentialBackoff(backoffDelay, maxRetryCount);
                    break;
                case NONE:
                    backoffPolicy = BackoffPolicy.noBackoff();
                    break;
                default:
                    throw new IllegalArgumentException("Received unknown backoff policy type " + bulkProcessorConfig.getFlushBackoffType());
            }
            builder.setBackoffPolicy(backoffPolicy);
            return builder;
        }
    };
}

Example 9 with RestHighLevelClient

use of org.elasticsearch.client.RestHighLevelClient in project flink by apache.

the class Elasticsearch7ApiCallBridge method createClient.

@Override
public RestHighLevelClient createClient(Map<String, String> clientConfig) {
    RestClientBuilder builder = RestClient.builder(httpHosts.toArray(new HttpHost[httpHosts.size()]));
    restClientFactory.configureRestClientBuilder(builder);
    RestHighLevelClient rhlClient = new RestHighLevelClient(builder);
    return rhlClient;
}

Example 10 with RestHighLevelClient

use of org.elasticsearch.client.RestHighLevelClient in project flink by apache.

the class Elasticsearch6ApiCallBridge method createClient.

@Override
public RestHighLevelClient createClient(Map<String, String> clientConfig) {
    RestClientBuilder builder = RestClient.builder(httpHosts.toArray(new HttpHost[httpHosts.size()]));
    restClientFactory.configureRestClientBuilder(builder);
    RestHighLevelClient rhlClient = new RestHighLevelClient(builder);
    return rhlClient;
}