Examples with Buffer org.komamitsu.fluency.buffer.Buffer used on opensource projects

Example 6 with Buffer

use of org.komamitsu.fluency.buffer.Buffer in project fluency by komamitsu.

the class FlusherTest method testAsyncFlusher.

@Test
void testAsyncFlusher() throws IOException, InterruptedException {
    flusherConfig.setFlushAttemptIntervalMillis(500);
    Buffer buffer = spy(new Buffer(bufferConfig, new JsonRecordFormatter()));
    Flusher flusher = new Flusher(flusherConfig, buffer, ingester);
    verify(buffer, times(0)).flush(eq(ingester), anyBoolean());
    verify(buffer, times(0)).flush(eq(ingester), anyBoolean());
    flusher.flush();
    TimeUnit.MILLISECONDS.sleep(50);
    verify(buffer, times(0)).flush(eq(ingester), eq(false));
    verify(buffer, times(1)).flush(eq(ingester), eq(true));
    TimeUnit.SECONDS.sleep(1);
    verify(buffer, atLeast(1)).flush(eq(ingester), eq(false));
    verify(buffer, atMost(3)).flush(eq(ingester), eq(false));
    verify(buffer, times(1)).flush(eq(ingester), eq(true));
    verify(buffer, times(0)).close();
    flusher.close();
    verify(buffer, times(1)).close();
    verify(buffer, atLeast(2)).flush(eq(ingester), eq(false));
    verify(buffer, atMost(3)).flush(eq(ingester), eq(false));
    verify(buffer, atLeast(2)).flush(eq(ingester), eq(true));
    verify(buffer, atMost(3)).flush(eq(ingester), eq(true));
}

Example 7 with Buffer

use of org.komamitsu.fluency.buffer.Buffer in project fluency by komamitsu.

the class FluencyTest method testWaitUntilFlushingAllBuffer.

@ParameterizedTest
@CsvSource({ "1, false", "3, true" })
void testWaitUntilFlushingAllBuffer(int waitUntilFlusherTerm, boolean expected) throws IOException, InterruptedException {
    flusherConfig.setFlushAttemptIntervalMillis(2000);
    Buffer buffer = new Buffer(bufferConfig, new JsonRecordFormatter());
    Flusher flusher = new Flusher(flusherConfig, buffer, ingester);
    try (Fluency fluency = new Fluency(buffer, flusher)) {
        fluency.emit("foo.bar", new HashMap<>());
        assertThat(fluency.waitUntilAllBufferFlushed(waitUntilFlusherTerm), is(expected));
    }
}

Example 8 with Buffer

use of org.komamitsu.fluency.buffer.Buffer in project fluency by komamitsu.

the class FluencyTest method testWaitUntilFlusherTerminated.

@ParameterizedTest
@CsvSource({ "1, false", "3, true" })
void testWaitUntilFlusherTerminated(int waitUntilFlusherTerm, boolean expected) throws IOException, InterruptedException {
    flusherConfig.setWaitUntilTerminated(1);
    // Wait before actually closing in Buffer
    int waitBeforeCloseMillis = 2000;
    Buffer buffer = spy(new Buffer(bufferConfig, new JsonRecordFormatter()));
    doAnswer((invocation) -> {
        long start = System.currentTimeMillis();
        try {
            TimeUnit.MILLISECONDS.sleep(waitBeforeCloseMillis);
        } catch (InterruptedException e) {
            long rest = waitBeforeCloseMillis - (System.currentTimeMillis() - start);
            if (rest > 0) {
                try {
                    TimeUnit.MILLISECONDS.sleep(rest);
                } catch (InterruptedException e1) {
                }
            }
        }
        return null;
    }).doCallRealMethod().when(buffer).close();
    Flusher flusher = new Flusher(flusherConfig, buffer, ingester);
    Fluency fluency = new Fluency(buffer, flusher);
    fluency.emit("foo.bar", new HashMap<>());
    fluency.close();
    assertThat(fluency.waitUntilFlusherTerminated(waitUntilFlusherTerm), is(expected));
}

Example 9 with Buffer

use of org.komamitsu.fluency.buffer.Buffer in project fluency by komamitsu.

the class FluencyTest method testBufferFullException.

@Test
public void testBufferFullException() throws IOException {
    final CountDownLatch latch = new CountDownLatch(1);
    StuckIngester stuckIngester = new StuckIngester(latch);
    bufferConfig.setChunkInitialSize(64);
    bufferConfig.setChunkExpandRatio(2);
    bufferConfig.setMaxBufferSize(256);
    bufferConfig.setChunkRetentionSize(196);
    flusherConfig.setFlushAttemptIntervalMillis(1000);
    Buffer buffer = new Buffer(bufferConfig, new JsonRecordFormatter());
    Flusher flusher = new Flusher(flusherConfig, buffer, stuckIngester);
    try (Fluency fluency = new Fluency(buffer, flusher)) {
        Map<String, Object> event = new HashMap<>();
        // '{"name":"xxxx"}' (length: 15 bytes)
        event.put("name", "xxxx");
        // 15 * (8 + 1) = 135
        for (int i = 0; i < 8; i++) {
            fluency.emit("tag", event);
        }
        try {
            fluency.emit("tag", event);
            fail();
        } catch (BufferFullException e) {
            assertTrue(true);
        } finally {
            latch.countDown();
        }
    }
}

Example 10 with Buffer

use of org.komamitsu.fluency.buffer.Buffer in project fluency by komamitsu.

the class FlusherTest method queueSizeShouldNotExceedLimit.

@Test
void queueSizeShouldNotExceedLimit() throws Exception {
    flusherConfig.setFlushAttemptIntervalMillis(200);
    Buffer buffer = new Buffer(bufferConfig, new JsonRecordFormatter());
    Flusher flusher = new Flusher(flusherConfig, buffer, ingester);
    BlockingQueue<Boolean> eventQueue = (BlockingQueue<Boolean>) extractValue(flusher, "eventQueue");
    int nonZeroCounts = 0;
    for (int index = 0; index < 10_000; index++) {
        flusher.flush();
        if (!eventQueue.isEmpty()) {
            nonZeroCounts++;
        }
        // The eventQueue will always have less that 16 elements (the default max size)
        assertTrue(eventQueue.size() <= 16);
    }
    // The eventQueue will be non empty at least a couple of times
    assertTrue(nonZeroCounts > 0);
    // Wait for sufficiently long (amount of time > queue poll wait) and the queue should be polled completely to become empty
    Thread.sleep(1000);
    assertEquals(0, eventQueue.size());
}