Examples with RestHighLevelClient org.elasticsearch.client.RestHighLevelClient used on opensource projects

Example 51 with RestHighLevelClient

use of org.elasticsearch.client.RestHighLevelClient in project flink by apache.

the class Elasticsearch6SinkBuilder method getBulkProcessorBuilderFactory.

@Override
protected BulkProcessorBuilderFactory getBulkProcessorBuilderFactory() {
    return new BulkProcessorBuilderFactory() {

        @Override
        public BulkProcessor.Builder apply(RestHighLevelClient client, BulkProcessorConfig bulkProcessorConfig, BulkProcessor.Listener listener) {
            BulkProcessor.Builder builder = BulkProcessor.builder(new // This cannot be inlined as a
            BulkRequestConsumerFactory() {

                // lambda because then
                // deserialization fails
                @Override
                public void accept(BulkRequest bulkRequest, ActionListener<BulkResponse> bulkResponseActionListener) {
                    client.bulkAsync(bulkRequest, bulkResponseActionListener);
                }
            }, listener);
            if (bulkProcessorConfig.getBulkFlushMaxActions() != -1) {
                builder.setBulkActions(bulkProcessorConfig.getBulkFlushMaxActions());
            }
            if (bulkProcessorConfig.getBulkFlushMaxMb() != -1) {
                builder.setBulkSize(new ByteSizeValue(bulkProcessorConfig.getBulkFlushMaxMb(), ByteSizeUnit.MB));
            }
            if (bulkProcessorConfig.getBulkFlushInterval() != -1) {
                builder.setFlushInterval(new TimeValue(bulkProcessorConfig.getBulkFlushInterval()));
            }
            BackoffPolicy backoffPolicy;
            final TimeValue backoffDelay = new TimeValue(bulkProcessorConfig.getBulkFlushBackOffDelay());
            final int maxRetryCount = bulkProcessorConfig.getBulkFlushBackoffRetries();
            switch(bulkProcessorConfig.getFlushBackoffType()) {
                case CONSTANT:
                    backoffPolicy = BackoffPolicy.constantBackoff(backoffDelay, maxRetryCount);
                    break;
                case EXPONENTIAL:
                    backoffPolicy = BackoffPolicy.exponentialBackoff(backoffDelay, maxRetryCount);
                    break;
                case NONE:
                    backoffPolicy = BackoffPolicy.noBackoff();
                    break;
                default:
                    throw new IllegalArgumentException("Received unknown backoff policy type " + bulkProcessorConfig.getFlushBackoffType());
            }
            builder.setBackoffPolicy(backoffPolicy);
            return builder;
        }
    };
}

Example 52 with RestHighLevelClient

use of org.elasticsearch.client.RestHighLevelClient in project flink by apache.

the class ElasticsearchDynamicSinkBaseITCase method testWritingDocumentsFromTableApi.

@Test
public void testWritingDocumentsFromTableApi() throws Exception {
    TableEnvironment tableEnvironment = TableEnvironment.create(EnvironmentSettings.inStreamingMode());
    String index = "table-api";
    tableEnvironment.executeSql("CREATE TABLE esTable (" + "a BIGINT NOT NULL,\n" + "b TIME,\n" + "c STRING NOT NULL,\n" + "d FLOAT,\n" + "e TINYINT NOT NULL,\n" + "f DATE,\n" + "g TIMESTAMP NOT NULL,\n" + "h as a + 2,\n" + "PRIMARY KEY (a, g) NOT ENFORCED\n" + ")\n" + "WITH (\n" + getConnectorSql(index) + ")");
    tableEnvironment.fromValues(row(1L, LocalTime.ofNanoOfDay(12345L * 1_000_000L), "ABCDE", 12.12f, (byte) 2, LocalDate.ofEpochDay(12345), LocalDateTime.parse("2012-12-12T12:12:12"))).executeInsert("esTable").await();
    RestHighLevelClient client = getClient();
    Map<String, Object> response = makeGetRequest(client, index, "1_2012-12-12T12:12:12");
    Map<Object, Object> expectedMap = new HashMap<>();
    expectedMap.put("a", 1);
    expectedMap.put("b", "00:00:12");
    expectedMap.put("c", "ABCDE");
    expectedMap.put("d", 12.12d);
    expectedMap.put("e", 2);
    expectedMap.put("f", "2003-10-20");
    expectedMap.put("g", "2012-12-12 12:12:12");
    Assertions.assertEquals(response, expectedMap);
}

Example 53 with RestHighLevelClient

use of org.elasticsearch.client.RestHighLevelClient in project flink by apache.

the class ElasticsearchDynamicSinkBaseITCase method testWritingDocumentsWithDynamicIndexFromSystemTime.

@Test
public void testWritingDocumentsWithDynamicIndexFromSystemTime() throws Exception {
    TableEnvironment tableEnvironment = TableEnvironment.create(EnvironmentSettings.inStreamingMode());
    DateTimeFormatter dateTimeFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd");
    tableEnvironment.getConfig().getConfiguration().setString("table.local-time-zone", "Asia/Shanghai");
    String dynamicIndex1 = "dynamic-index-" + dateTimeFormatter.format(LocalDateTime.now(ZoneId.of("Asia/Shanghai"))) + "_index";
    String index = "dynamic-index-{now()|yyyy-MM-dd}_index";
    tableEnvironment.executeSql("CREATE TABLE esTable (" + "a BIGINT NOT NULL,\n" + "b TIMESTAMP NOT NULL,\n" + "PRIMARY KEY (a) NOT ENFORCED\n" + ")\n" + "WITH (\n" + getConnectorSql(index) + ")");
    String dynamicIndex2 = "dynamic-index-" + dateTimeFormatter.format(LocalDateTime.now(ZoneId.of("Asia/Shanghai"))) + "_index";
    tableEnvironment.fromValues(row(1L, LocalDateTime.parse("2012-12-12T12:12:12"))).executeInsert("esTable").await();
    RestHighLevelClient client = getClient();
    Map<String, Object> response;
    try {
        response = makeGetRequest(client, dynamicIndex1, "1");
    } catch (ElasticsearchStatusException e) {
        if (e.status() == RestStatus.NOT_FOUND) {
            response = makeGetRequest(client, dynamicIndex2, "1");
        } else {
            throw e;
        }
    }
    Map<Object, Object> expectedMap = new HashMap<>();
    expectedMap.put("a", 1);
    expectedMap.put("b", "2012-12-12 12:12:12");
    Assertions.assertEquals(response, expectedMap);
}

Example 54 with RestHighLevelClient

use of org.elasticsearch.client.RestHighLevelClient in project flink by apache.

the class ElasticsearchDynamicSinkBaseITCase method testWritingDocumentsNoPrimaryKey.

@Test
public void testWritingDocumentsNoPrimaryKey() throws Exception {
    TableEnvironment tableEnvironment = TableEnvironment.create(EnvironmentSettings.inStreamingMode());
    String index = "no-primary-key";
    tableEnvironment.executeSql("CREATE TABLE esTable (" + "a BIGINT NOT NULL,\n" + "b TIME,\n" + "c STRING NOT NULL,\n" + "d FLOAT,\n" + "e TINYINT NOT NULL,\n" + "f DATE,\n" + "g TIMESTAMP NOT NULL\n" + ")\n" + "WITH (\n" + getConnectorSql(index) + ")");
    tableEnvironment.fromValues(row(1L, LocalTime.ofNanoOfDay(12345L * 1_000_000L), "ABCDE", 12.12f, (byte) 2, LocalDate.ofEpochDay(12345), LocalDateTime.parse("2012-12-12T12:12:12")), row(2L, LocalTime.ofNanoOfDay(12345L * 1_000_000L), "FGHIJK", 13.13f, (byte) 4, LocalDate.ofEpochDay(12345), LocalDateTime.parse("2013-12-12T13:13:13"))).executeInsert("esTable").await();
    RestHighLevelClient client = getClient();
    // search API does not return documents that were not indexed, we might need to query
    // the index a few times
    Deadline deadline = Deadline.fromNow(Duration.ofSeconds(30));
    SearchHits hits;
    do {
        hits = makeSearchRequest(client, index);
        if (getTotalSearchHits(hits) < 2) {
            Thread.sleep(200);
        }
    } while (getTotalSearchHits(hits) < 2 && deadline.hasTimeLeft());
    if (getTotalSearchHits(hits) < 2) {
        throw new AssertionError("Could not retrieve results from Elasticsearch.");
    }
    HashSet<Map<String, Object>> resultSet = new HashSet<>();
    resultSet.add(hits.getAt(0).getSourceAsMap());
    resultSet.add(hits.getAt(1).getSourceAsMap());
    Map<Object, Object> expectedMap1 = new HashMap<>();
    expectedMap1.put("a", 1);
    expectedMap1.put("b", "00:00:12");
    expectedMap1.put("c", "ABCDE");
    expectedMap1.put("d", 12.12d);
    expectedMap1.put("e", 2);
    expectedMap1.put("f", "2003-10-20");
    expectedMap1.put("g", "2012-12-12 12:12:12");
    Map<Object, Object> expectedMap2 = new HashMap<>();
    expectedMap2.put("a", 2);
    expectedMap2.put("b", "00:00:12");
    expectedMap2.put("c", "FGHIJK");
    expectedMap2.put("d", 13.13d);
    expectedMap2.put("e", 4);
    expectedMap2.put("f", "2003-10-20");
    expectedMap2.put("g", "2013-12-12 13:13:13");
    HashSet<Map<Object, Object>> expectedSet = new HashSet<>();
    expectedSet.add(expectedMap1);
    expectedSet.add(expectedMap2);
    Assertions.assertEquals(resultSet, expectedSet);
}

Example 55 with RestHighLevelClient

use of org.elasticsearch.client.RestHighLevelClient in project flink by apache.

the class ElasticsearchDynamicSinkBaseITCase method testWritingDocumentsWithDynamicIndex.

@Test
public void testWritingDocumentsWithDynamicIndex() throws Exception {
    TableEnvironment tableEnvironment = TableEnvironment.create(EnvironmentSettings.inStreamingMode());
    String index = "dynamic-index-{b|yyyy-MM-dd}";
    tableEnvironment.executeSql("CREATE TABLE esTable (" + "a BIGINT NOT NULL,\n" + "b TIMESTAMP NOT NULL,\n" + "PRIMARY KEY (a) NOT ENFORCED\n" + ")\n" + "WITH (\n" + getConnectorSql(index) + ")");
    tableEnvironment.fromValues(row(1L, LocalDateTime.parse("2012-12-12T12:12:12"))).executeInsert("esTable").await();
    RestHighLevelClient client = getClient();
    Map<String, Object> response = makeGetRequest(client, "dynamic-index-2012-12-12", "1");
    Map<Object, Object> expectedMap = new HashMap<>();
    expectedMap.put("a", 1);
    expectedMap.put("b", "2012-12-12 12:12:12");
    Assertions.assertEquals(response, expectedMap);
}