Examples with Fluency org.komamitsu.fluency.Fluency used on opensource projects

Example 1 with Fluency

use of org.komamitsu.fluency.Fluency in project fluency by komamitsu.

the class FluencyTestWithMockServer method testFluencyUsingAsyncFlusher.

@ParameterizedTest
@MethodSource("optionsProvider")
void testFluencyUsingAsyncFlusher(final Options options) throws Exception {
    testFluencyBase(localPorts -> {
        FluentdSender sender;
        int fluentdPort = localPorts.get(0);
        if (options.failover) {
            int secondaryFluentdPort = localPorts.get(1);
            if (options.sslEnabled) {
                sender = getDoubleSSLSender(fluentdPort, secondaryFluentdPort);
            } else {
                sender = getDoubleTCPSender(fluentdPort, secondaryFluentdPort);
            }
        } else {
            if (options.sslEnabled) {
                sender = getSingleSSLSender(fluentdPort);
            } else {
                sender = getSingleTCPSender(fluentdPort);
            }
        }
        FluentdIngester.Config ingesterConfig = new FluentdIngester.Config();
        if (options.ackResponse) {
            ingesterConfig.setAckResponseMode(true);
        }
        Buffer.Config bufferConfig = new Buffer.Config();
        if (options.smallBuffer) {
            bufferConfig.setChunkRetentionSize(SMALL_BUF_SIZE);
            bufferConfig.setMaxBufferSize(SMALL_BUF_SIZE + 1);
        }
        if (options.fileBackup) {
            bufferConfig.setFileBackupDir(TMPDIR);
            bufferConfig.setFileBackupPrefix("testFluencyUsingAsyncFlusher" + options.hashCode());
        }
        if (options.jvmHeap) {
            bufferConfig.setJvmHeapBufferMode(true);
        }
        Flusher.Config flusherConfig = new Flusher.Config();
        flusherConfig.setWaitUntilBufferFlushed(10);
        flusherConfig.setWaitUntilTerminated(10);
        Buffer buffer = new Buffer(bufferConfig, new FluentdRecordFormatter(new FluentdRecordFormatter.Config()));
        Flusher flusher = new Flusher(flusherConfig, buffer, new FluentdIngester(ingesterConfig, sender));
        return new Fluency(buffer, flusher);
    }, options);
}

Example 2 with Fluency

use of org.komamitsu.fluency.Fluency in project fluency by komamitsu.

the class FluencyTestWithMockServer method testFluencyBase.

private void testFluencyBase(final FluencyFactory fluencyFactory, final Options options) throws Exception {
    LOG.info("testFluencyBase starts: options={}", options);
    final ArrayList<Integer> localPorts = new ArrayList<>();
    final MockFluentdServer fluentd = new MockFluentdServer(options.sslEnabled);
    fluentd.start();
    final MockFluentdServer secondaryFluentd = new MockFluentdServer(options.sslEnabled, fluentd);
    secondaryFluentd.start();
    localPorts.add(fluentd.getLocalPort());
    localPorts.add(secondaryFluentd.getLocalPort());
    final AtomicReference<Fluency> fluency = new AtomicReference<>(fluencyFactory.generate(localPorts));
    if (options.fileBackup) {
        fluency.get().clearBackupFiles();
    }
    final ObjectMapper objectMapper = new ObjectMapper(new MessagePackFactory());
    final int maxNameLen = 200;
    final HashMap<Integer, String> nameLenTable = new HashMap<>(maxNameLen);
    for (int i = 1; i <= maxNameLen; i++) {
        StringBuilder stringBuilder = new StringBuilder();
        for (int j = 0; j < i; j++) {
            stringBuilder.append('x');
        }
        nameLenTable.put(i, stringBuilder.toString());
    }
    final AtomicLong ageEventsSum = new AtomicLong();
    final AtomicLong nameEventsLength = new AtomicLong();
    final AtomicLong tag0EventsCounter = new AtomicLong();
    final AtomicLong tag1EventsCounter = new AtomicLong();
    final AtomicLong tag2EventsCounter = new AtomicLong();
    final AtomicLong tag3EventsCounter = new AtomicLong();
    try {
        final Random random = new Random();
        int concurrency = 10;
        final int reqNum = 6000;
        long start = System.currentTimeMillis();
        final CountDownLatch latch = new CountDownLatch(concurrency);
        final AtomicBoolean shouldFailOver = new AtomicBoolean(true);
        final AtomicBoolean shouldStopFluentd = new AtomicBoolean(true);
        final AtomicBoolean shouldStopFluency = new AtomicBoolean(true);
        final CountDownLatch fluentdCloseWaitLatch = new CountDownLatch(concurrency);
        final CountDownLatch fluencyCloseWaitLatch = new CountDownLatch(concurrency);
        ExecutorService es = Executors.newCachedThreadPool();
        for (int i = 0; i < concurrency; i++) {
            es.execute(() -> {
                for (int i1 = 0; i1 < reqNum; i1++) {
                    if (Thread.currentThread().isInterrupted()) {
                        LOG.info("Interrupted...");
                        break;
                    }
                    if (options.failover) {
                        if (i1 == reqNum / 2) {
                            if (shouldFailOver.getAndSet(false)) {
                                LOG.info("Failing over...");
                                try {
                                    secondaryFluentd.stop();
                                } catch (IOException e) {
                                    LOG.warn("Failed to stop secondary fluentd", e);
                                }
                            }
                        }
                    } else if (options.fileBackup) {
                        if (i1 == reqNum / 2) {
                            if (shouldStopFluentd.getAndSet(false)) {
                                LOG.info("Stopping Fluentd...");
                                try {
                                    fluentd.stop();
                                    secondaryFluentd.stop();
                                } catch (IOException e) {
                                    LOG.warn("Failed to stop Fluentd", e);
                                }
                            }
                            fluentdCloseWaitLatch.countDown();
                            try {
                                assertTrue(fluentdCloseWaitLatch.await(20, TimeUnit.SECONDS));
                            } catch (InterruptedException e) {
                                LOG.warn("Interrupted", e);
                            }
                            if (shouldStopFluency.getAndSet(false)) {
                                LOG.info("Stopping Fluency...");
                                try {
                                    fluency.get().close();
                                    TimeUnit.SECONDS.sleep(2);
                                } catch (Exception e) {
                                    LOG.warn("Failed to stop Fluency", e);
                                }
                                LOG.info("Restarting Fluentd...");
                                try {
                                    fluentd.start();
                                    secondaryFluentd.start();
                                    LOG.info("Restarting Fluency...");
                                    fluency.set(fluencyFactory.generate(Arrays.asList(fluentd.getLocalPort(), secondaryFluentd.getLocalPort())));
                                    TimeUnit.SECONDS.sleep(2);
                                } catch (Exception e) {
                                    LOG.warn("Failed to restart Fluentd", e);
                                }
                            }
                            fluencyCloseWaitLatch.countDown();
                            try {
                                assertTrue(fluencyCloseWaitLatch.await(20, TimeUnit.SECONDS));
                            } catch (InterruptedException e) {
                                LOG.warn("Interrupted", e);
                            }
                        }
                    }
                    int tagNum = i1 % 4;
                    final String tag = String.format("foodb%d.bartbl%d", tagNum, tagNum);
                    switch(tagNum) {
                        case 0:
                            tag0EventsCounter.incrementAndGet();
                            break;
                        case 1:
                            tag1EventsCounter.incrementAndGet();
                            break;
                        case 2:
                            tag2EventsCounter.incrementAndGet();
                            break;
                        case 3:
                            tag3EventsCounter.incrementAndGet();
                            break;
                        default:
                            throw new RuntimeException("Never reach here");
                    }
                    int rand = random.nextInt(maxNameLen);
                    final Map<String, Object> hashMap = new HashMap<String, Object>();
                    String name = nameLenTable.get(rand + 1);
                    nameEventsLength.addAndGet(name.length());
                    hashMap.put("name", name);
                    rand = random.nextInt(100);
                    int age = rand;
                    ageEventsSum.addAndGet(age);
                    hashMap.put("age", age);
                    hashMap.put("comment", "hello, world");
                    hashMap.put("rate", 1.23);
                    try {
                        Exception exception = null;
                        for (int retry = 0; retry < 10; retry++) {
                            try {
                                switch(options.emitType) {
                                    case MAP:
                                        {
                                            fluency.get().emit(tag, hashMap);
                                            break;
                                        }
                                    case MAP_WITH_EVENT_TIME:
                                        {
                                            EventTime eventTime = EventTime.fromEpochMilli(System.currentTimeMillis());
                                            fluency.get().emit(tag, eventTime, hashMap);
                                            break;
                                        }
                                    case MSGPACK_MAP_VALUE_BYTES:
                                        {
                                            byte[] bytes = objectMapper.writeValueAsBytes(hashMap);
                                            fluency.get().emit(tag, bytes, 0, bytes.length);
                                            break;
                                        }
                                    case MSGPACK_MAP_VALUE_BYTES_WITH_EVENT_TIME:
                                        {
                                            EventTime eventTime = EventTime.fromEpochMilli(System.currentTimeMillis());
                                            byte[] bytes = objectMapper.writeValueAsBytes(hashMap);
                                            fluency.get().emit(tag, eventTime, bytes, 0, bytes.length);
                                            break;
                                        }
                                    case MSGPACK_MAP_VALUE_BYTEBUFFER:
                                        {
                                            byte[] bytes = objectMapper.writeValueAsBytes(hashMap);
                                            fluency.get().emit(tag, ByteBuffer.wrap(bytes));
                                            break;
                                        }
                                    case MSGPACK_MAP_VALUE_BYTEBUFFER_WITH_EVENT_TIME:
                                        {
                                            EventTime eventTime = EventTime.fromEpochMilli(System.currentTimeMillis());
                                            byte[] bytes = objectMapper.writeValueAsBytes(hashMap);
                                            fluency.get().emit(tag, eventTime, ByteBuffer.wrap(bytes));
                                            break;
                                        }
                                }
                                exception = null;
                                break;
                            } catch (Exception e) {
                                exception = e;
                                try {
                                    TimeUnit.SECONDS.sleep(1);
                                } catch (InterruptedException e1) {
                                }
                            }
                        }
                        if (exception != null) {
                            throw exception;
                        }
                    } catch (Exception e) {
                        LOG.warn("Exception occurred", e);
                    }
                }
                latch.countDown();
            });
        }
        if (!latch.await(60, TimeUnit.SECONDS)) {
            fail("Sending all requests is timed out");
        }
        if (options.closeInsteadOfFlush) {
            fluency.get().close();
        } else {
            fluency.get().flush();
            fluency.get().waitUntilAllBufferFlushed(20);
        }
        fluentd.waitUntilEventsStop();
        fluentd.stop();
        secondaryFluentd.waitUntilEventsStop();
        secondaryFluentd.stop();
        if (options.failover) {
            assertThat(fluentd.connectCounter.get(), is(greaterThan(0L)));
            assertThat(fluentd.connectCounter.get(), is(lessThanOrEqualTo(10L)));
            assertThat(fluentd.closeCounter.get(), is(greaterThan(0L)));
            assertThat(fluentd.closeCounter.get(), is(lessThanOrEqualTo(10L)));
        } else {
            assertThat(fluentd.connectCounter.get(), is(greaterThan(0L)));
            assertThat(fluentd.connectCounter.get(), is(lessThanOrEqualTo(2L)));
            if (options.closeInsteadOfFlush) {
                assertThat(fluentd.closeCounter.get(), is(greaterThan(0L)));
            } else {
                assertThat(fluentd.closeCounter.get(), is(0L));
            }
            assertThat(fluentd.closeCounter.get(), is(lessThanOrEqualTo(2L)));
        }
        assertEquals((long) concurrency * reqNum, fluentd.ageEventsCounter.get());
        assertEquals(ageEventsSum.get(), fluentd.ageEventsSum.get());
        assertEquals((long) concurrency * reqNum, fluentd.nameEventsCounter.get());
        assertEquals(nameEventsLength.get(), fluentd.nameEventsLength.get());
        assertEquals(tag0EventsCounter.get(), fluentd.tag0EventsCounter.get());
        assertEquals(tag1EventsCounter.get(), fluentd.tag1EventsCounter.get());
        assertEquals(tag2EventsCounter.get(), fluentd.tag2EventsCounter.get());
        assertEquals(tag3EventsCounter.get(), fluentd.tag3EventsCounter.get());
        System.out.println(System.currentTimeMillis() - start);
    } finally {
        fluency.get().close();
        fluentd.stop();
        secondaryFluentd.stop();
    }
}

Example 3 with Fluency

use of org.komamitsu.fluency.Fluency in project fluency by komamitsu.

the class FluencyBuilderForFluentdTest method buildWithSslAndCustomPort.

@Test
void buildWithSslAndCustomPort() throws IOException {
    FluencyBuilderForFluentd builder = new FluencyBuilderForFluentd();
    builder.setSslEnabled(true);
    try (Fluency fluency = builder.build(54321)) {
        assertBuffer(fluency.getBuffer());
        assertFlusher(fluency.getFlusher());
        assertDefaultFluentdSender((FluentdSender) fluency.getFlusher().getIngester().getSender(), "127.0.0.1", 54321, SSLSender.class);
    }
}

Example 4 with Fluency

use of org.komamitsu.fluency.Fluency in project fluency by komamitsu.

the class FluencyBuilderForFluentdTest method buildWithSsl.

@Test
void buildWithSsl() throws IOException {
    FluencyBuilderForFluentd builder = new FluencyBuilderForFluentd();
    builder.setSslEnabled(true);
    try (Fluency fluency = builder.build()) {
        assertBuffer(fluency.getBuffer());
        assertFlusher(fluency.getFlusher());
        assertDefaultFluentdSender((FluentdSender) fluency.getFlusher().getIngester().getSender(), "127.0.0.1", 24224, SSLSender.class);
    }
}

Example 5 with Fluency

use of org.komamitsu.fluency.Fluency in project fluency by komamitsu.

the class FluencyBuilderForFluentdTest method buildWithComplexConfig.

@Test
void buildWithComplexConfig() throws IOException {
    String tmpdir = System.getProperty("java.io.tmpdir");
    assertThat(tmpdir, is(notNullValue()));
    FluencyBuilderForFluentd builder = new FluencyBuilderForFluentd();
    builder.setFlushAttemptIntervalMillis(200);
    builder.setMaxBufferSize(Long.MAX_VALUE);
    builder.setBufferChunkInitialSize(7 * 1024 * 1024);
    builder.setBufferChunkRetentionSize(13 * 1024 * 1024);
    builder.setBufferChunkRetentionTimeMillis(19 * 1000);
    builder.setJvmHeapBufferMode(true);
    builder.setSenderMaxRetryCount(99);
    builder.setSenderBaseRetryIntervalMillis(20);
    builder.setSenderMaxRetryIntervalMillis(100000);
    builder.setConnectionTimeoutMilli(12345);
    builder.setReadTimeoutMilli(9876);
    builder.setAckResponseMode(true);
    builder.setWaitUntilBufferFlushed(42);
    builder.setWaitUntilFlusherTerminated(24);
    builder.setFileBackupDir(tmpdir);
    try (Fluency fluency = builder.build(Arrays.asList(new InetSocketAddress("333.333.333.333", 11111), new InetSocketAddress("444.444.444.444", 22222)))) {
        assertThat(fluency.getBuffer(), instanceOf(Buffer.class));
        Buffer buffer = fluency.getBuffer();
        assertThat(buffer.getMaxBufferSize(), is(Long.MAX_VALUE));
        assertThat(buffer.getFileBackupDir(), is(tmpdir));
        assertThat(buffer.bufferFormatType(), is("packed_forward"));
        assertThat(buffer.getChunkRetentionTimeMillis(), is(19 * 1000));
        assertThat(buffer.getChunkExpandRatio(), is(2f));
        assertThat(buffer.getChunkInitialSize(), is(7 * 1024 * 1024));
        assertThat(buffer.getChunkRetentionSize(), is(13 * 1024 * 1024));
        assertThat(buffer.getJvmHeapBufferMode(), is(true));
        Flusher flusher = fluency.getFlusher();
        assertThat(flusher.isTerminated(), is(false));
        assertThat(flusher.getFlushAttemptIntervalMillis(), is(200));
        assertThat(flusher.getWaitUntilBufferFlushed(), is(42));
        assertThat(flusher.getWaitUntilTerminated(), is(24));
        assertThat(flusher.getIngester().getSender(), instanceOf(RetryableSender.class));
        RetryableSender retryableSender = (RetryableSender) flusher.getIngester().getSender();
        assertThat(retryableSender.getRetryStrategy(), instanceOf(ExponentialBackOffRetryStrategy.class));
        ExponentialBackOffRetryStrategy retryStrategy = (ExponentialBackOffRetryStrategy) retryableSender.getRetryStrategy();
        assertThat(retryStrategy.getMaxRetryCount(), is(99));
        assertThat(retryStrategy.getBaseIntervalMillis(), is(20));
        assertThat(retryStrategy.getMaxIntervalMillis(), is(100000));
        assertThat(retryableSender.getBaseSender(), instanceOf(MultiSender.class));
        MultiSender multiSender = (MultiSender) retryableSender.getBaseSender();
        assertThat(multiSender.getSenders().size(), is(2));
        assertThat(multiSender.getSenders().get(0), instanceOf(TCPSender.class));
        {
            TCPSender sender = (TCPSender) multiSender.getSenders().get(0);
            assertThat(sender.getHost(), is("333.333.333.333"));
            assertThat(sender.getPort(), is(11111));
            assertThat(sender.getConnectionTimeoutMilli(), is(12345));
            assertThat(sender.getReadTimeoutMilli(), is(9876));
            FailureDetector failureDetector = sender.getFailureDetector();
            assertThat(failureDetector.getFailureIntervalMillis(), is(3 * 1000));
            assertThat(failureDetector.getFailureDetectStrategy(), instanceOf(PhiAccrualFailureDetectStrategy.class));
            assertThat(failureDetector.getHeartbeater(), instanceOf(TCPHeartbeater.class));
            assertThat(failureDetector.getHeartbeater().getHost(), is("333.333.333.333"));
            assertThat(failureDetector.getHeartbeater().getPort(), is(11111));
            assertThat(failureDetector.getHeartbeater().getIntervalMillis(), is(1000));
        }
        assertThat(multiSender.getSenders().get(1), instanceOf(TCPSender.class));
        {
            TCPSender sender = (TCPSender) multiSender.getSenders().get(1);
            assertThat(sender.getHost(), is("444.444.444.444"));
            assertThat(sender.getPort(), is(22222));
            assertThat(sender.getConnectionTimeoutMilli(), is(12345));
            assertThat(sender.getReadTimeoutMilli(), is(9876));
            FailureDetector failureDetector = sender.getFailureDetector();
            assertThat(failureDetector.getFailureIntervalMillis(), is(3 * 1000));
            assertThat(failureDetector.getFailureDetectStrategy(), instanceOf(PhiAccrualFailureDetectStrategy.class));
            assertThat(failureDetector.getHeartbeater(), instanceOf(TCPHeartbeater.class));
            assertThat(failureDetector.getHeartbeater().getHost(), is("444.444.444.444"));
            assertThat(failureDetector.getHeartbeater().getPort(), is(22222));
            assertThat(failureDetector.getHeartbeater().getIntervalMillis(), is(1000));
        }
    }
}