Examples with RestHighLevelClient org.graylog.shaded.elasticsearch7.org.elasticsearch.client.RestHighLevelClient used on opensource projects

Example 41 with RestHighLevelClient

use of org.graylog.shaded.elasticsearch7.org.elasticsearch.client.RestHighLevelClient in project presto by prestodb.

the class ElasticsearchClient method createClient.

private static RestHighLevelClient createClient(ElasticsearchConfig config, Optional<AwsSecurityConfig> awsSecurityConfig) {
    RestClientBuilder builder = RestClient.builder(new HttpHost(config.getHost(), config.getPort(), config.isTlsEnabled() ? "https" : "http")).setMaxRetryTimeoutMillis((int) config.getMaxRetryTime().toMillis());
    builder.setHttpClientConfigCallback(ignored -> {
        RequestConfig requestConfig = RequestConfig.custom().setConnectTimeout(toIntExact(config.getConnectTimeout().toMillis())).setSocketTimeout(toIntExact(config.getRequestTimeout().toMillis())).build();
        IOReactorConfig reactorConfig = IOReactorConfig.custom().setIoThreadCount(config.getHttpThreadCount()).build();
        // the client builder passed to the call-back is configured to use system properties, which makes it
        // impossible to configure concurrency settings, so we need to build a new one from scratch
        HttpAsyncClientBuilder clientBuilder = HttpAsyncClientBuilder.create().setDefaultRequestConfig(requestConfig).setDefaultIOReactorConfig(reactorConfig).setMaxConnPerRoute(config.getMaxHttpConnections()).setMaxConnTotal(config.getMaxHttpConnections());
        if (config.isTlsEnabled()) {
            buildSslContext(config.getKeystorePath(), config.getKeystorePassword(), config.getTrustStorePath(), config.getTruststorePassword()).ifPresent(clientBuilder::setSSLContext);
            if (config.isVerifyHostnames()) {
                clientBuilder.setSSLHostnameVerifier(NoopHostnameVerifier.INSTANCE);
            }
        }
        awsSecurityConfig.ifPresent(securityConfig -> clientBuilder.addInterceptorLast(new AwsRequestSigner(securityConfig.getRegion(), getAwsCredentialsProvider(securityConfig))));
        return clientBuilder;
    });
    return new RestHighLevelClient(builder);
}

Example 42 with RestHighLevelClient

use of org.graylog.shaded.elasticsearch7.org.elasticsearch.client.RestHighLevelClient in project flink by apache.

the class ElasticsearchSinkBaseITCase method createRestHighLevelClient.

private RestHighLevelClient createRestHighLevelClient() {
    final CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
    credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(ELASTICSEARCH_USER, ELASTICSEARCH_PASSWORD));
    return new RestHighLevelClient(RestClient.builder(HttpHost.create(getElasticsearchHttpHostAddress())).setHttpClientConfigCallback(httpClientBuilder -> httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider)));
}

Example 43 with RestHighLevelClient

use of org.graylog.shaded.elasticsearch7.org.elasticsearch.client.RestHighLevelClient in project flink by apache.

the class Elasticsearch6SinkBuilder method getBulkProcessorBuilderFactory.

@Override
protected BulkProcessorBuilderFactory getBulkProcessorBuilderFactory() {
    return new BulkProcessorBuilderFactory() {

        @Override
        public BulkProcessor.Builder apply(RestHighLevelClient client, BulkProcessorConfig bulkProcessorConfig, BulkProcessor.Listener listener) {
            BulkProcessor.Builder builder = BulkProcessor.builder(new // This cannot be inlined as a
            BulkRequestConsumerFactory() {

                // lambda because then
                // deserialization fails
                @Override
                public void accept(BulkRequest bulkRequest, ActionListener<BulkResponse> bulkResponseActionListener) {
                    client.bulkAsync(bulkRequest, bulkResponseActionListener);
                }
            }, listener);
            if (bulkProcessorConfig.getBulkFlushMaxActions() != -1) {
                builder.setBulkActions(bulkProcessorConfig.getBulkFlushMaxActions());
            }
            if (bulkProcessorConfig.getBulkFlushMaxMb() != -1) {
                builder.setBulkSize(new ByteSizeValue(bulkProcessorConfig.getBulkFlushMaxMb(), ByteSizeUnit.MB));
            }
            if (bulkProcessorConfig.getBulkFlushInterval() != -1) {
                builder.setFlushInterval(new TimeValue(bulkProcessorConfig.getBulkFlushInterval()));
            }
            BackoffPolicy backoffPolicy;
            final TimeValue backoffDelay = new TimeValue(bulkProcessorConfig.getBulkFlushBackOffDelay());
            final int maxRetryCount = bulkProcessorConfig.getBulkFlushBackoffRetries();
            switch(bulkProcessorConfig.getFlushBackoffType()) {
                case CONSTANT:
                    backoffPolicy = BackoffPolicy.constantBackoff(backoffDelay, maxRetryCount);
                    break;
                case EXPONENTIAL:
                    backoffPolicy = BackoffPolicy.exponentialBackoff(backoffDelay, maxRetryCount);
                    break;
                case NONE:
                    backoffPolicy = BackoffPolicy.noBackoff();
                    break;
                default:
                    throw new IllegalArgumentException("Received unknown backoff policy type " + bulkProcessorConfig.getFlushBackoffType());
            }
            builder.setBackoffPolicy(backoffPolicy);
            return builder;
        }
    };
}

Example 44 with RestHighLevelClient

use of org.graylog.shaded.elasticsearch7.org.elasticsearch.client.RestHighLevelClient in project flink by apache.

the class ElasticsearchDynamicSinkBaseITCase method testWritingDocumentsFromTableApi.

@Test
public void testWritingDocumentsFromTableApi() throws Exception {
    TableEnvironment tableEnvironment = TableEnvironment.create(EnvironmentSettings.inStreamingMode());
    String index = "table-api";
    tableEnvironment.executeSql("CREATE TABLE esTable (" + "a BIGINT NOT NULL,\n" + "b TIME,\n" + "c STRING NOT NULL,\n" + "d FLOAT,\n" + "e TINYINT NOT NULL,\n" + "f DATE,\n" + "g TIMESTAMP NOT NULL,\n" + "h as a + 2,\n" + "PRIMARY KEY (a, g) NOT ENFORCED\n" + ")\n" + "WITH (\n" + getConnectorSql(index) + ")");
    tableEnvironment.fromValues(row(1L, LocalTime.ofNanoOfDay(12345L * 1_000_000L), "ABCDE", 12.12f, (byte) 2, LocalDate.ofEpochDay(12345), LocalDateTime.parse("2012-12-12T12:12:12"))).executeInsert("esTable").await();
    RestHighLevelClient client = getClient();
    Map<String, Object> response = makeGetRequest(client, index, "1_2012-12-12T12:12:12");
    Map<Object, Object> expectedMap = new HashMap<>();
    expectedMap.put("a", 1);
    expectedMap.put("b", "00:00:12");
    expectedMap.put("c", "ABCDE");
    expectedMap.put("d", 12.12d);
    expectedMap.put("e", 2);
    expectedMap.put("f", "2003-10-20");
    expectedMap.put("g", "2012-12-12 12:12:12");
    Assertions.assertEquals(response, expectedMap);
}

Example 45 with RestHighLevelClient

use of org.graylog.shaded.elasticsearch7.org.elasticsearch.client.RestHighLevelClient in project flink by apache.

the class ElasticsearchDynamicSinkBaseITCase method testWritingDocumentsWithDynamicIndexFromSystemTime.

@Test
public void testWritingDocumentsWithDynamicIndexFromSystemTime() throws Exception {
    TableEnvironment tableEnvironment = TableEnvironment.create(EnvironmentSettings.inStreamingMode());
    DateTimeFormatter dateTimeFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd");
    tableEnvironment.getConfig().getConfiguration().setString("table.local-time-zone", "Asia/Shanghai");
    String dynamicIndex1 = "dynamic-index-" + dateTimeFormatter.format(LocalDateTime.now(ZoneId.of("Asia/Shanghai"))) + "_index";
    String index = "dynamic-index-{now()|yyyy-MM-dd}_index";
    tableEnvironment.executeSql("CREATE TABLE esTable (" + "a BIGINT NOT NULL,\n" + "b TIMESTAMP NOT NULL,\n" + "PRIMARY KEY (a) NOT ENFORCED\n" + ")\n" + "WITH (\n" + getConnectorSql(index) + ")");
    String dynamicIndex2 = "dynamic-index-" + dateTimeFormatter.format(LocalDateTime.now(ZoneId.of("Asia/Shanghai"))) + "_index";
    tableEnvironment.fromValues(row(1L, LocalDateTime.parse("2012-12-12T12:12:12"))).executeInsert("esTable").await();
    RestHighLevelClient client = getClient();
    Map<String, Object> response;
    try {
        response = makeGetRequest(client, dynamicIndex1, "1");
    } catch (ElasticsearchStatusException e) {
        if (e.status() == RestStatus.NOT_FOUND) {
            response = makeGetRequest(client, dynamicIndex2, "1");
        } else {
            throw e;
        }
    }
    Map<Object, Object> expectedMap = new HashMap<>();
    expectedMap.put("a", 1);
    expectedMap.put("b", "2012-12-12 12:12:12");
    Assertions.assertEquals(response, expectedMap);
}