Examples with MockTCPSender org.komamitsu.fluency.sender.MockTCPSender used on opensource projects

Example 1 with MockTCPSender

use of org.komamitsu.fluency.sender.MockTCPSender in project fluency by komamitsu.

the class FluencyTest method testIsTerminated.

@Test
public void testIsTerminated() throws IOException, InterruptedException {
    Sender sender = new MockTCPSender(24224);
    TestableBuffer.Config bufferConfig = new TestableBuffer.Config();
    {
        Flusher.Instantiator flusherConfig = new AsyncFlusher.Config();
        Fluency fluency = new Fluency.Builder(sender).setBufferConfig(bufferConfig).setFlusherConfig(flusherConfig).build();
        assertFalse(fluency.isTerminated());
        fluency.close();
        TimeUnit.SECONDS.sleep(1);
        assertTrue(fluency.isTerminated());
    }
    {
        Flusher.Instantiator flusherConfig = new SyncFlusher.Config();
        Fluency fluency = new Fluency.Builder(sender).setBufferConfig(bufferConfig).setFlusherConfig(flusherConfig).build();
        assertFalse(fluency.isTerminated());
        fluency.close();
        TimeUnit.SECONDS.sleep(1);
        assertTrue(fluency.isTerminated());
    }
}

Example 2 with MockTCPSender

use of org.komamitsu.fluency.sender.MockTCPSender in project fluency by komamitsu.

the class BufferTestHelper method baseTestMessageBuffer.

void baseTestMessageBuffer(final int loopCount, final boolean multiTags, final boolean syncFlush, final boolean eventTime, final Buffer buffer) throws IOException, InterruptedException {
    assertThat(buffer.getBufferUsage(), is(0f));
    assertThat(buffer.getAllocatedSize(), is(0L));
    assertThat(buffer.getBufferedDataSize(), is(0L));
    final int concurrency = 4;
    final CountDownLatch latch = new CountDownLatch(concurrency);
    final MockTCPSender sender = new MockTCPSender(24229);
    Runnable emitTask = new Runnable() {

        @Override
        public void run() {
            try {
                for (int i = 0; i < loopCount; i++) {
                    HashMap<String, Object> data = new HashMap<String, Object>();
                    data.put("name", String.format("komamitsu%06d", i));
                    data.put("age", i);
                    data.put("comment", i % 31 == 0 ? longStr : "hello");
                    String tag = multiTags ? String.format("foodb%d.bartbl%d", i % 4, i % 4) : "foodb.bartbl";
                    if (eventTime) {
                        buffer.append(tag, new EventTime((int) (System.currentTimeMillis() / 1000), 999999999), data);
                    } else {
                        buffer.append(tag, System.currentTimeMillis(), data);
                    }
                }
                latch.countDown();
            } catch (Exception e) {
                e.printStackTrace();
            }
        }
    };
    final ExecutorService flushService = Executors.newSingleThreadExecutor();
    if (!syncFlush) {
        flushService.execute(new Runnable() {

            @Override
            public void run() {
                while (!flushService.isShutdown()) {
                    try {
                        TimeUnit.MILLISECONDS.sleep(100L);
                        buffer.flush(sender, false);
                    } catch (InterruptedException e) {
                        Thread.currentThread().interrupt();
                    } catch (IOException e) {
                        e.printStackTrace();
                        return;
                    }
                }
            }
        });
    }
    long start = System.currentTimeMillis();
    final ExecutorService executorService = Executors.newFixedThreadPool(concurrency);
    for (int i = 0; i < concurrency; i++) {
        executorService.execute(emitTask);
    }
    assertTrue(latch.await(10, TimeUnit.SECONDS));
    assertThat(buffer.getBufferUsage(), is(greaterThan(0f)));
    assertThat(buffer.getAllocatedSize(), is(greaterThan(0L)));
    assertThat(buffer.getBufferedDataSize(), is(greaterThan(0L)));
    buffer.flush(sender, true);
    buffer.close();
    long end = System.currentTimeMillis();
    executorService.shutdown();
    executorService.awaitTermination(10, TimeUnit.SECONDS);
    if (!executorService.isTerminated()) {
        executorService.shutdownNow();
    }
    flushService.shutdown();
    flushService.awaitTermination(10, TimeUnit.SECONDS);
    if (!flushService.isTerminated()) {
        flushService.shutdownNow();
    }
    // Just in case
    buffer.close();
    assertThat(buffer.getBufferUsage(), is(0f));
    assertThat(buffer.getAllocatedSize(), is(0L));
    assertThat(buffer.getBufferedDataSize(), is(0L));
    int totalLoopCount = concurrency * loopCount;
    int recordCount = 0;
    ObjectMapper objectMapper = new ObjectMapper(new MessagePackFactory());
    objectMapper.disable(JsonParser.Feature.AUTO_CLOSE_SOURCE);
    for (MockTCPSender.Event event : sender.getEvents()) {
        byte[] bytes = event.getAllBytes();
        MessageUnpacker messageUnpacker = MessagePack.newDefaultUnpacker(bytes);
        assertEquals(3, messageUnpacker.unpackArrayHeader());
        String tag = messageUnpacker.unpackString();
        byte[] payload = messageUnpacker.readPayload(messageUnpacker.unpackBinaryHeader());
        messageUnpacker = MessagePack.newDefaultUnpacker(payload);
        while (messageUnpacker.hasNext()) {
            assertEquals(2, messageUnpacker.unpackArrayHeader());
            ImmutableValue timestamp = messageUnpacker.unpackValue();
            int size = messageUnpacker.unpackMapHeader();
            assertEquals(3, size);
            Map<String, Object> data = new HashMap<String, Object>();
            for (int i = 0; i < size; i++) {
                String key = messageUnpacker.unpackString();
                ImmutableValue value = messageUnpacker.unpackValue();
                if (value.isStringValue()) {
                    data.put(key, value.asStringValue().asString());
                } else if (value.isIntegerValue()) {
                    data.put(key, value.asIntegerValue().asInt());
                }
            }
            analyzeResult(tag, timestamp, data, start, end, eventTime);
            recordCount++;
        }
    }
    assertEquals(totalLoopCount, recordCount);
    if (multiTags) {
        assertEquals(4, tagCounts.size());
        for (int i = 0; i < 4; i++) {
            int count = tagCounts.get(String.format("foodb%d.bartbl%d", i, i));
            assertTrue(totalLoopCount / 4 - 4 <= count && count <= totalLoopCount / 4 + 4);
        }
    } else {
        assertEquals(1, tagCounts.size());
        int count = tagCounts.get("foodb.bartbl");
        assertEquals(totalLoopCount, count);
    }
    assertEquals("komamitsu000000", minName);
    assertEquals(String.format("komamitsu%06d", loopCount - 1), maxName);
    assertEquals(0, minAge);
    assertEquals(loopCount - 1, maxAge);
    assertTrue(totalLoopCount / 31 - 5 <= longCommentCount && longCommentCount <= totalLoopCount / 31 + 5);
}

Example 3 with MockTCPSender

use of org.komamitsu.fluency.sender.MockTCPSender in project fluency by komamitsu.

the class PackedForwardBufferTest method testGetBufferedDataSize.

@Test
public void testGetBufferedDataSize() throws IOException, InterruptedException {
    PackedForwardBuffer buffer = new PackedForwardBuffer.Config().setChunkInitialSize(256 * 1024).createInstance();
    assertThat(buffer.getBufferedDataSize(), is(0L));
    Map<String, Object> map = new HashMap<String, Object>();
    map.put("name", "komamitsu");
    for (int i = 0; i < 10; i++) {
        buffer.append("foo.bar", new Date().getTime(), map);
    }
    assertThat(buffer.getBufferedDataSize(), is(greaterThan(0L)));
    assertThat(buffer.getBufferedDataSize(), is(lessThan(512L)));
    MockTCPSender sender = new MockTCPSender(24224);
    buffer.flush(sender, true);
    assertThat(buffer.getBufferedDataSize(), is(0L));
}

Example 4 with MockTCPSender

use of org.komamitsu.fluency.sender.MockTCPSender in project fluency by komamitsu.

the class AsyncFlusherTest method testAsyncFlusher.

@Test
public void testAsyncFlusher() throws IOException, InterruptedException {
    TestableBuffer buffer = new TestableBuffer.Config().createInstance();
    MockTCPSender sender = new MockTCPSender(24225);
    AsyncFlusher.Config config = new AsyncFlusher.Config();
    config.setFlushIntervalMillis(500);
    Flusher flusher = config.createInstance(buffer, sender);
    assertEquals(0, buffer.getFlushCount().get());
    flusher.onUpdate();
    assertEquals(0, buffer.getFlushCount().get());
    flusher.flush();
    TimeUnit.MILLISECONDS.sleep(50);
    assertEquals(0, buffer.getFlushCount().get());
    assertEquals(1, buffer.getForceFlushCount().get());
    TimeUnit.SECONDS.sleep(1);
    int flushCount = buffer.getFlushCount().get();
    assertTrue(1 <= flushCount && flushCount <= 3);
    int forceFlushCount = buffer.getForceFlushCount().get();
    assertEquals(1, forceFlushCount);
    assertEquals(0, buffer.getCloseCount().get());
    flusher.close();
    assertEquals(1, buffer.getCloseCount().get());
    assertThat(buffer.getFlushCount().get(), is(greaterThanOrEqualTo(2)));
    assertThat(buffer.getFlushCount().get(), is(lessThanOrEqualTo(3)));
    assertThat(buffer.getForceFlushCount().get(), is(greaterThanOrEqualTo(2)));
    assertThat(buffer.getForceFlushCount().get(), is(lessThanOrEqualTo(3)));
}

Example 5 with MockTCPSender

use of org.komamitsu.fluency.sender.MockTCPSender in project fluency by komamitsu.

the class SyncFlusherTest method testSyncFlusher.

@Test
public void testSyncFlusher() throws IOException, InterruptedException {
    TestableBuffer buffer = new TestableBuffer.Config().createInstance();
    MockTCPSender sender = new MockTCPSender(24225);
    SyncFlusher.Config config = new SyncFlusher.Config();
    assertEquals(600, config.getFlushIntervalMillis());
    Flusher flusher = config.createInstance(buffer, sender);
    flusher.flush();
    flusher.flush();
    flusher.flush();
    assertEquals(0, buffer.getFlushCount().get());
    assertEquals(3, buffer.getForceFlushCount().get());
    flusher.onUpdate();
    flusher.onUpdate();
    flusher.onUpdate();
    assertEquals(0, buffer.getFlushCount().get());
    assertEquals(3, buffer.getForceFlushCount().get());
    TimeUnit.SECONDS.sleep(1);
    flusher.onUpdate();
    assertEquals(1, buffer.getFlushCount().get());
    assertEquals(3, buffer.getForceFlushCount().get());
    assertEquals(0, buffer.getCloseCount().get());
    flusher.close();
    assertEquals(1, buffer.getCloseCount().get());
    assertEquals(1, buffer.getFlushCount().get());
    assertEquals(3 + 1, buffer.getForceFlushCount().get());
}