Examples with Flusher org.komamitsu.fluency.flusher.Flusher used on opensource projects

Example 1 with Flusher

use of org.komamitsu.fluency.flusher.Flusher in project fluency by komamitsu.

the class FluencyTestWithMockServer method testFluencyUsingAsyncFlusher.

@ParameterizedTest
@MethodSource("optionsProvider")
void testFluencyUsingAsyncFlusher(final Options options) throws Exception {
    testFluencyBase(localPorts -> {
        FluentdSender sender;
        int fluentdPort = localPorts.get(0);
        if (options.failover) {
            int secondaryFluentdPort = localPorts.get(1);
            if (options.sslEnabled) {
                sender = getDoubleSSLSender(fluentdPort, secondaryFluentdPort);
            } else {
                sender = getDoubleTCPSender(fluentdPort, secondaryFluentdPort);
            }
        } else {
            if (options.sslEnabled) {
                sender = getSingleSSLSender(fluentdPort);
            } else {
                sender = getSingleTCPSender(fluentdPort);
            }
        }
        FluentdIngester.Config ingesterConfig = new FluentdIngester.Config();
        if (options.ackResponse) {
            ingesterConfig.setAckResponseMode(true);
        }
        Buffer.Config bufferConfig = new Buffer.Config();
        if (options.smallBuffer) {
            bufferConfig.setChunkRetentionSize(SMALL_BUF_SIZE);
            bufferConfig.setMaxBufferSize(SMALL_BUF_SIZE + 1);
        }
        if (options.fileBackup) {
            bufferConfig.setFileBackupDir(TMPDIR);
            bufferConfig.setFileBackupPrefix("testFluencyUsingAsyncFlusher" + options.hashCode());
        }
        if (options.jvmHeap) {
            bufferConfig.setJvmHeapBufferMode(true);
        }
        Flusher.Config flusherConfig = new Flusher.Config();
        flusherConfig.setWaitUntilBufferFlushed(10);
        flusherConfig.setWaitUntilTerminated(10);
        Buffer buffer = new Buffer(bufferConfig, new FluentdRecordFormatter(new FluentdRecordFormatter.Config()));
        Flusher flusher = new Flusher(flusherConfig, buffer, new FluentdIngester(ingesterConfig, sender));
        return new Fluency(buffer, flusher);
    }, options);
}

Example 2 with Flusher

use of org.komamitsu.fluency.flusher.Flusher in project fluency by komamitsu.

the class FluencyBuilderForFluentdTest method buildWithComplexConfig.

@Test
void buildWithComplexConfig() throws IOException {
    String tmpdir = System.getProperty("java.io.tmpdir");
    assertThat(tmpdir, is(notNullValue()));
    FluencyBuilderForFluentd builder = new FluencyBuilderForFluentd();
    builder.setFlushAttemptIntervalMillis(200);
    builder.setMaxBufferSize(Long.MAX_VALUE);
    builder.setBufferChunkInitialSize(7 * 1024 * 1024);
    builder.setBufferChunkRetentionSize(13 * 1024 * 1024);
    builder.setBufferChunkRetentionTimeMillis(19 * 1000);
    builder.setJvmHeapBufferMode(true);
    builder.setSenderMaxRetryCount(99);
    builder.setSenderBaseRetryIntervalMillis(20);
    builder.setSenderMaxRetryIntervalMillis(100000);
    builder.setConnectionTimeoutMilli(12345);
    builder.setReadTimeoutMilli(9876);
    builder.setAckResponseMode(true);
    builder.setWaitUntilBufferFlushed(42);
    builder.setWaitUntilFlusherTerminated(24);
    builder.setFileBackupDir(tmpdir);
    try (Fluency fluency = builder.build(Arrays.asList(new InetSocketAddress("333.333.333.333", 11111), new InetSocketAddress("444.444.444.444", 22222)))) {
        assertThat(fluency.getBuffer(), instanceOf(Buffer.class));
        Buffer buffer = fluency.getBuffer();
        assertThat(buffer.getMaxBufferSize(), is(Long.MAX_VALUE));
        assertThat(buffer.getFileBackupDir(), is(tmpdir));
        assertThat(buffer.bufferFormatType(), is("packed_forward"));
        assertThat(buffer.getChunkRetentionTimeMillis(), is(19 * 1000));
        assertThat(buffer.getChunkExpandRatio(), is(2f));
        assertThat(buffer.getChunkInitialSize(), is(7 * 1024 * 1024));
        assertThat(buffer.getChunkRetentionSize(), is(13 * 1024 * 1024));
        assertThat(buffer.getJvmHeapBufferMode(), is(true));
        Flusher flusher = fluency.getFlusher();
        assertThat(flusher.isTerminated(), is(false));
        assertThat(flusher.getFlushAttemptIntervalMillis(), is(200));
        assertThat(flusher.getWaitUntilBufferFlushed(), is(42));
        assertThat(flusher.getWaitUntilTerminated(), is(24));
        assertThat(flusher.getIngester().getSender(), instanceOf(RetryableSender.class));
        RetryableSender retryableSender = (RetryableSender) flusher.getIngester().getSender();
        assertThat(retryableSender.getRetryStrategy(), instanceOf(ExponentialBackOffRetryStrategy.class));
        ExponentialBackOffRetryStrategy retryStrategy = (ExponentialBackOffRetryStrategy) retryableSender.getRetryStrategy();
        assertThat(retryStrategy.getMaxRetryCount(), is(99));
        assertThat(retryStrategy.getBaseIntervalMillis(), is(20));
        assertThat(retryStrategy.getMaxIntervalMillis(), is(100000));
        assertThat(retryableSender.getBaseSender(), instanceOf(MultiSender.class));
        MultiSender multiSender = (MultiSender) retryableSender.getBaseSender();
        assertThat(multiSender.getSenders().size(), is(2));
        assertThat(multiSender.getSenders().get(0), instanceOf(TCPSender.class));
        {
            TCPSender sender = (TCPSender) multiSender.getSenders().get(0);
            assertThat(sender.getHost(), is("333.333.333.333"));
            assertThat(sender.getPort(), is(11111));
            assertThat(sender.getConnectionTimeoutMilli(), is(12345));
            assertThat(sender.getReadTimeoutMilli(), is(9876));
            FailureDetector failureDetector = sender.getFailureDetector();
            assertThat(failureDetector.getFailureIntervalMillis(), is(3 * 1000));
            assertThat(failureDetector.getFailureDetectStrategy(), instanceOf(PhiAccrualFailureDetectStrategy.class));
            assertThat(failureDetector.getHeartbeater(), instanceOf(TCPHeartbeater.class));
            assertThat(failureDetector.getHeartbeater().getHost(), is("333.333.333.333"));
            assertThat(failureDetector.getHeartbeater().getPort(), is(11111));
            assertThat(failureDetector.getHeartbeater().getIntervalMillis(), is(1000));
        }
        assertThat(multiSender.getSenders().get(1), instanceOf(TCPSender.class));
        {
            TCPSender sender = (TCPSender) multiSender.getSenders().get(1);
            assertThat(sender.getHost(), is("444.444.444.444"));
            assertThat(sender.getPort(), is(22222));
            assertThat(sender.getConnectionTimeoutMilli(), is(12345));
            assertThat(sender.getReadTimeoutMilli(), is(9876));
            FailureDetector failureDetector = sender.getFailureDetector();
            assertThat(failureDetector.getFailureIntervalMillis(), is(3 * 1000));
            assertThat(failureDetector.getFailureDetectStrategy(), instanceOf(PhiAccrualFailureDetectStrategy.class));
            assertThat(failureDetector.getHeartbeater(), instanceOf(TCPHeartbeater.class));
            assertThat(failureDetector.getHeartbeater().getHost(), is("444.444.444.444"));
            assertThat(failureDetector.getHeartbeater().getPort(), is(22222));
            assertThat(failureDetector.getHeartbeater().getIntervalMillis(), is(1000));
        }
    }
}

Example 3 with Flusher

use of org.komamitsu.fluency.flusher.Flusher in project fluency by komamitsu.

the class FluencyBuilderForTreasureDataTest method buildWithAllCustomConfig.

@Test
void buildWithAllCustomConfig() throws IOException {
    String tmpdir = System.getProperty("java.io.tmpdir");
    assertNotNull(tmpdir);
    FluencyBuilderForTreasureData builder = new FluencyBuilderForTreasureData();
    builder.setFlushAttemptIntervalMillis(200);
    builder.setMaxBufferSize(Long.MAX_VALUE);
    builder.setBufferChunkInitialSize(7 * 1024 * 1024);
    builder.setBufferChunkRetentionSize(13 * 1024 * 1024);
    builder.setBufferChunkRetentionTimeMillis(19 * 1000);
    builder.setJvmHeapBufferMode(true);
    builder.setWaitUntilBufferFlushed(42);
    builder.setWaitUntilFlusherTerminated(24);
    builder.setFileBackupDir(tmpdir);
    builder.setSenderRetryIntervalMillis(1234);
    builder.setSenderMaxRetryIntervalMillis(345678);
    builder.setSenderRetryFactor(3.14f);
    builder.setSenderRetryMax(17);
    builder.setSenderWorkBufSize(123456);
    ;
    try (Fluency fluency = builder.build(APIKEY)) {
        assertEquals(Buffer.class, fluency.getBuffer().getClass());
        Buffer buffer = fluency.getBuffer();
        assertEquals(Long.MAX_VALUE, buffer.getMaxBufferSize());
        assertEquals(tmpdir, buffer.getFileBackupDir());
        assertEquals("packed_forward", buffer.bufferFormatType());
        assertEquals(19 * 1000, buffer.getChunkRetentionTimeMillis());
        assertEquals(2f, buffer.getChunkExpandRatio());
        assertEquals(7 * 1024 * 1024, buffer.getChunkInitialSize());
        assertEquals(13 * 1024 * 1024, buffer.getChunkRetentionSize());
        assertTrue(buffer.getJvmHeapBufferMode());
        Flusher flusher = fluency.getFlusher();
        assertFalse(flusher.isTerminated());
        assertEquals(200, flusher.getFlushAttemptIntervalMillis());
        assertEquals(42, flusher.getWaitUntilBufferFlushed());
        assertEquals(24, flusher.getWaitUntilTerminated());
        assertEquals(TreasureDataSender.class, flusher.getIngester().getSender().getClass());
        TreasureDataSender sender = (TreasureDataSender) flusher.getIngester().getSender();
        assertEquals(1234, sender.getRetryInternalMs());
        assertEquals(345678, sender.getMaxRetryInternalMs());
        assertEquals(3.14f, sender.getRetryFactor());
        assertEquals(17, sender.getRetryMax());
        assertEquals(123456, sender.getWorkBufSize());
    }
}

Example 4 with Flusher

use of org.komamitsu.fluency.flusher.Flusher in project fluency by komamitsu.

the class FluencyTest method testGetAllocatedBufferSize.

@Test
void testGetAllocatedBufferSize() throws IOException {
    Buffer.Config bufferConfig = new Buffer.Config();
    bufferConfig.setChunkInitialSize(1024);
    Buffer buffer = new Buffer(bufferConfig, new JsonRecordFormatter());
    Flusher flusher = new Flusher(flusherConfig, buffer, ingester);
    try (Fluency fluency = new Fluency(buffer, flusher)) {
        assertThat(fluency.getAllocatedBufferSize(), is(0L));
        fluency.emit("foodb.bartbl", ImmutableMap.of("comment", "hello, world"));
        assertThat(fluency.getAllocatedBufferSize(), is(1024L));
    }
}

Example 5 with Flusher

use of org.komamitsu.fluency.flusher.Flusher in project fluency by komamitsu.

the class FluencyTest method testIsTerminated.

@Test
void testIsTerminated() throws IOException, InterruptedException {
    Buffer buffer = new Buffer(bufferConfig, new JsonRecordFormatter());
    Flusher flusher = new Flusher(flusherConfig, buffer, ingester);
    try (Fluency fluency = new Fluency(buffer, flusher)) {
        assertFalse(fluency.isTerminated());
        fluency.close();
        TimeUnit.SECONDS.sleep(1);
        assertTrue(fluency.isTerminated());
    }
}