Examples with ElasticToxiproxiContainer org.apache.pulsar.io.elasticsearch.testcontainers.ElasticToxiproxiContainer used on opensource projects

Example 1 with ElasticToxiproxiContainer

use of org.apache.pulsar.io.elasticsearch.testcontainers.ElasticToxiproxiContainer in project pulsar by apache.

the class ElasticSearchClientTests method testBulkRetry.

@Test
public void testBulkRetry() throws Exception {
    try (ElasticToxiproxiContainer toxiproxy = new ElasticToxiproxiContainer(container, network)) {
        toxiproxy.start();
        final String index = "indexbulktest-" + UUID.randomUUID();
        ElasticSearchConfig config = new ElasticSearchConfig().setElasticSearchUrl("http://" + toxiproxy.getHttpHostAddress()).setIndexName(index).setBulkEnabled(true).setMaxRetries(1000).setBulkActions(2).setRetryBackoffInMs(100).setBulkFlushIntervalInMs(-1);
        try (ElasticSearchClient client = new ElasticSearchClient(config)) {
            try {
                assertTrue(client.createIndexIfNeeded(index));
                MockRecord<GenericObject> mockRecord = new MockRecord<>();
                client.bulkIndex(mockRecord, Pair.of("1", "{\"a\":1}"));
                client.bulkIndex(mockRecord, Pair.of("2", "{\"a\":2}"));
                assertEquals(mockRecord.acked, 2);
                assertEquals(mockRecord.failed, 0);
                assertEquals(client.totalHits(index), 2);
                log.info("starting the toxic");
                toxiproxy.getProxy().setConnectionCut(false);
                toxiproxy.getProxy().toxics().latency("elasticpause", ToxicDirection.DOWNSTREAM, 15000);
                toxiproxy.removeToxicAfterDelay("elasticpause", 15000);
                client.bulkIndex(mockRecord, Pair.of("3", "{\"a\":3}"));
                assertEquals(mockRecord.acked, 2);
                assertEquals(mockRecord.failed, 0);
                assertEquals(client.totalHits(index), 2);
                client.flush();
                assertEquals(mockRecord.acked, 3);
                assertEquals(mockRecord.failed, 0);
                assertEquals(client.totalHits(index), 3);
            } finally {
                client.delete(index);
            }
        }
    }
}

Example 2 with ElasticToxiproxiContainer

use of org.apache.pulsar.io.elasticsearch.testcontainers.ElasticToxiproxiContainer in project pulsar by yahoo.

the class ElasticSearchClientTests method testBulkBlocking.

@Test
public void testBulkBlocking() throws Exception {
    try (ElasticToxiproxiContainer toxiproxy = new ElasticToxiproxiContainer(container, network)) {
        toxiproxy.start();
        final String index = "indexblocking-" + UUID.randomUUID();
        ElasticSearchConfig config = new ElasticSearchConfig().setElasticSearchUrl("http://" + toxiproxy.getHttpHostAddress()).setIndexName(index).setBulkEnabled(true).setMaxRetries(1000).setBulkActions(2).setBulkConcurrentRequests(2).setRetryBackoffInMs(100).setBulkFlushIntervalInMs(10000);
        try (ElasticSearchClient client = new ElasticSearchClient(config)) {
            assertTrue(client.createIndexIfNeeded(index));
            try {
                MockRecord<GenericObject> mockRecord = new MockRecord<>();
                for (int i = 1; i <= 5; i++) {
                    client.bulkIndex(mockRecord, Pair.of(Integer.toString(i), "{\"a\":" + i + "}"));
                }
                Awaitility.await().untilAsserted(() -> {
                    assertThat("acked record", mockRecord.acked, greaterThanOrEqualTo(4));
                    assertEquals(mockRecord.failed, 0);
                    assertThat("totalHits", client.getRestClient().totalHits(index), greaterThanOrEqualTo(4L));
                });
                client.flush();
                Awaitility.await().untilAsserted(() -> {
                    assertEquals(mockRecord.failed, 0);
                    assertEquals(mockRecord.acked, 5);
                    assertEquals(client.getRestClient().totalHits(index), 5);
                });
                log.info("starting the toxic");
                toxiproxy.getProxy().setConnectionCut(false);
                toxiproxy.getProxy().toxics().latency("elasticpause", ToxicDirection.DOWNSTREAM, 30000);
                toxiproxy.removeToxicAfterDelay("elasticpause", 30000);
                long start = System.currentTimeMillis();
                // 11th bulkIndex is blocking because we have 2 pending requests, and the 3rd request is blocked.
                for (int i = 6; i <= 15; i++) {
                    client.bulkIndex(mockRecord, Pair.of(Integer.toString(i), "{\"a\":" + i + "}"));
                    log.info("{} index {}", System.currentTimeMillis(), i);
                }
                long elapsed = System.currentTimeMillis() - start;
                log.info("elapsed = {}", elapsed);
                // bulkIndex was blocking while elasticsearch was down or busy
                assertTrue(elapsed > 29000);
                Awaitility.await().untilAsserted(() -> {
                    assertEquals(mockRecord.acked, 15);
                    assertEquals(mockRecord.failed, 0);
                });
            } finally {
                client.getRestClient().deleteIndex(index);
            }
        }
    }
}

Example 3 with ElasticToxiproxiContainer

use of org.apache.pulsar.io.elasticsearch.testcontainers.ElasticToxiproxiContainer in project incubator-pulsar by apache.

the class ElasticSearchClientTests method testBulkBlocking.

@Test
public void testBulkBlocking() throws Exception {
    try (ElasticToxiproxiContainer toxiproxy = new ElasticToxiproxiContainer(container, network)) {
        toxiproxy.start();
        final String index = "indexblocking-" + UUID.randomUUID();
        ElasticSearchConfig config = new ElasticSearchConfig().setElasticSearchUrl("http://" + toxiproxy.getHttpHostAddress()).setIndexName(index).setBulkEnabled(true).setMaxRetries(1000).setBulkActions(2).setBulkConcurrentRequests(2).setRetryBackoffInMs(100).setBulkFlushIntervalInMs(10000);
        try (ElasticSearchClient client = new ElasticSearchClient(config)) {
            assertTrue(client.createIndexIfNeeded(index));
            try {
                MockRecord<GenericObject> mockRecord = new MockRecord<>();
                for (int i = 1; i <= 5; i++) {
                    client.bulkIndex(mockRecord, Pair.of(Integer.toString(i), "{\"a\":" + i + "}"));
                }
                Awaitility.await().untilAsserted(() -> {
                    assertThat("acked record", mockRecord.acked, greaterThanOrEqualTo(4));
                    assertEquals(mockRecord.failed, 0);
                    assertThat("totalHits", client.getRestClient().totalHits(index), greaterThanOrEqualTo(4L));
                });
                client.flush();
                Awaitility.await().untilAsserted(() -> {
                    assertEquals(mockRecord.failed, 0);
                    assertEquals(mockRecord.acked, 5);
                    assertEquals(client.getRestClient().totalHits(index), 5);
                });
                log.info("starting the toxic");
                toxiproxy.getProxy().setConnectionCut(false);
                toxiproxy.getProxy().toxics().latency("elasticpause", ToxicDirection.DOWNSTREAM, 30000);
                toxiproxy.removeToxicAfterDelay("elasticpause", 30000);
                long start = System.currentTimeMillis();
                // 11th bulkIndex is blocking because we have 2 pending requests, and the 3rd request is blocked.
                for (int i = 6; i <= 15; i++) {
                    client.bulkIndex(mockRecord, Pair.of(Integer.toString(i), "{\"a\":" + i + "}"));
                    log.info("{} index {}", System.currentTimeMillis(), i);
                }
                long elapsed = System.currentTimeMillis() - start;
                log.info("elapsed = {}", elapsed);
                // bulkIndex was blocking while elasticsearch was down or busy
                assertTrue(elapsed > 29000);
                Awaitility.await().untilAsserted(() -> {
                    assertEquals(mockRecord.acked, 15);
                    assertEquals(mockRecord.failed, 0);
                });
            } finally {
                client.getRestClient().deleteIndex(index);
            }
        }
    }
}

Example 4 with ElasticToxiproxiContainer

use of org.apache.pulsar.io.elasticsearch.testcontainers.ElasticToxiproxiContainer in project incubator-pulsar by apache.

the class ElasticSearchClientTests method testBulkRetry.

@Test
public void testBulkRetry() throws Exception {
    try (ElasticToxiproxiContainer toxiproxy = new ElasticToxiproxiContainer(container, network)) {
        toxiproxy.start();
        final String index = "indexbulktest-" + UUID.randomUUID();
        ElasticSearchConfig config = new ElasticSearchConfig().setElasticSearchUrl("http://" + toxiproxy.getHttpHostAddress()).setIndexName(index).setBulkEnabled(true).setMaxRetries(1000).setBulkActions(2).setRetryBackoffInMs(100).setBulkFlushIntervalInMs(-1);
        try (ElasticSearchClient client = new ElasticSearchClient(config)) {
            try {
                assertTrue(client.createIndexIfNeeded(index));
                MockRecord<GenericObject> mockRecord = new MockRecord<>();
                client.bulkIndex(mockRecord, Pair.of("1", "{\"a\":1}"));
                client.bulkIndex(mockRecord, Pair.of("2", "{\"a\":2}"));
                assertEquals(mockRecord.acked, 2);
                assertEquals(mockRecord.failed, 0);
                assertEquals(client.getRestClient().totalHits(index), 2);
                log.info("starting the toxic");
                toxiproxy.getProxy().setConnectionCut(false);
                toxiproxy.getProxy().toxics().latency("elasticpause", ToxicDirection.DOWNSTREAM, 15000);
                toxiproxy.removeToxicAfterDelay("elasticpause", 15000);
                client.bulkIndex(mockRecord, Pair.of("3", "{\"a\":3}"));
                assertEquals(mockRecord.acked, 2);
                assertEquals(mockRecord.failed, 0);
                assertEquals(client.getRestClient().totalHits(index), 2);
                client.flush();
                assertEquals(mockRecord.acked, 3);
                assertEquals(mockRecord.failed, 0);
                assertEquals(client.getRestClient().totalHits(index), 3);
            } finally {
                client.getRestClient().deleteIndex(index);
            }
        }
    }
}

Example 5 with ElasticToxiproxiContainer

use of org.apache.pulsar.io.elasticsearch.testcontainers.ElasticToxiproxiContainer in project pulsar by apache.

the class ElasticSearchClientTests method testBulkBlocking.

@Test
public void testBulkBlocking() throws Exception {
    try (ElasticToxiproxiContainer toxiproxy = new ElasticToxiproxiContainer(container, network)) {
        toxiproxy.start();
        final String index = "indexblocking-" + UUID.randomUUID();
        ElasticSearchConfig config = new ElasticSearchConfig().setElasticSearchUrl("http://" + toxiproxy.getHttpHostAddress()).setIndexName(index).setBulkEnabled(true).setMaxRetries(1000).setBulkActions(2).setBulkConcurrentRequests(2).setRetryBackoffInMs(100).setBulkFlushIntervalInMs(10000);
        try (ElasticSearchClient client = new ElasticSearchClient(config)) {
            assertTrue(client.createIndexIfNeeded(index));
            try {
                MockRecord<GenericObject> mockRecord = new MockRecord<>();
                for (int i = 1; i <= 5; i++) {
                    client.bulkIndex(mockRecord, Pair.of(Integer.toString(i), "{\"a\":" + i + "}"));
                }
                Awaitility.await().untilAsserted(() -> {
                    assertThat("acked record", mockRecord.acked, greaterThanOrEqualTo(4));
                    assertEquals(mockRecord.failed, 0);
                    assertThat("totalHits", client.totalHits(index), greaterThanOrEqualTo(4L));
                });
                client.flush();
                Awaitility.await().untilAsserted(() -> {
                    assertEquals(mockRecord.failed, 0);
                    assertEquals(mockRecord.acked, 5);
                    assertEquals(client.totalHits(index), 5);
                });
                log.info("starting the toxic");
                toxiproxy.getProxy().setConnectionCut(false);
                toxiproxy.getProxy().toxics().latency("elasticpause", ToxicDirection.DOWNSTREAM, 30000);
                toxiproxy.removeToxicAfterDelay("elasticpause", 30000);
                long start = System.currentTimeMillis();
                // 11th bulkIndex is blocking because we have 2 pending requests, and the 3rd request is blocked.
                for (int i = 6; i <= 15; i++) {
                    client.bulkIndex(mockRecord, Pair.of(Integer.toString(i), "{\"a\":" + i + "}"));
                    log.info("{} index {}", System.currentTimeMillis(), i);
                }
                long elapsed = System.currentTimeMillis() - start;
                log.info("elapsed = {}", elapsed);
                // bulkIndex was blocking while elasticsearch was down or busy
                assertTrue(elapsed > 29000);
                Thread.sleep(3000L);
                assertEquals(mockRecord.acked, 15);
                assertEquals(mockRecord.failed, 0);
                assertEquals(client.records.size(), 0);
            } finally {
                client.delete(index);
            }
        }
    }
}