Examples with EventTime org.komamitsu.fluency.EventTime used on opensource projects

Example 1 with EventTime

use of org.komamitsu.fluency.EventTime in project fluency by komamitsu.

the class BufferTestHelper method baseTestMessageBuffer.

void baseTestMessageBuffer(final int loopCount, final boolean multiTags, final boolean syncFlush, final boolean eventTime, final Buffer buffer) throws IOException, InterruptedException {
    assertThat(buffer.getBufferUsage(), is(0f));
    assertThat(buffer.getAllocatedSize(), is(0L));
    assertThat(buffer.getBufferedDataSize(), is(0L));
    final int concurrency = 4;
    final CountDownLatch latch = new CountDownLatch(concurrency);
    final MockTCPSender sender = new MockTCPSender(24229);
    Runnable emitTask = new Runnable() {

        @Override
        public void run() {
            try {
                for (int i = 0; i < loopCount; i++) {
                    HashMap<String, Object> data = new HashMap<String, Object>();
                    data.put("name", String.format("komamitsu%06d", i));
                    data.put("age", i);
                    data.put("comment", i % 31 == 0 ? longStr : "hello");
                    String tag = multiTags ? String.format("foodb%d.bartbl%d", i % 4, i % 4) : "foodb.bartbl";
                    if (eventTime) {
                        buffer.append(tag, new EventTime((int) (System.currentTimeMillis() / 1000), 999999999), data);
                    } else {
                        buffer.append(tag, System.currentTimeMillis(), data);
                    }
                }
                latch.countDown();
            } catch (Exception e) {
                e.printStackTrace();
            }
        }
    };
    final ExecutorService flushService = Executors.newSingleThreadExecutor();
    if (!syncFlush) {
        flushService.execute(new Runnable() {

            @Override
            public void run() {
                while (!flushService.isShutdown()) {
                    try {
                        TimeUnit.MILLISECONDS.sleep(100L);
                        buffer.flush(sender, false);
                    } catch (InterruptedException e) {
                        Thread.currentThread().interrupt();
                    } catch (IOException e) {
                        e.printStackTrace();
                        return;
                    }
                }
            }
        });
    }
    long start = System.currentTimeMillis();
    final ExecutorService executorService = Executors.newFixedThreadPool(concurrency);
    for (int i = 0; i < concurrency; i++) {
        executorService.execute(emitTask);
    }
    assertTrue(latch.await(10, TimeUnit.SECONDS));
    assertThat(buffer.getBufferUsage(), is(greaterThan(0f)));
    assertThat(buffer.getAllocatedSize(), is(greaterThan(0L)));
    assertThat(buffer.getBufferedDataSize(), is(greaterThan(0L)));
    buffer.flush(sender, true);
    buffer.close();
    long end = System.currentTimeMillis();
    executorService.shutdown();
    executorService.awaitTermination(10, TimeUnit.SECONDS);
    if (!executorService.isTerminated()) {
        executorService.shutdownNow();
    }
    flushService.shutdown();
    flushService.awaitTermination(10, TimeUnit.SECONDS);
    if (!flushService.isTerminated()) {
        flushService.shutdownNow();
    }
    // Just in case
    buffer.close();
    assertThat(buffer.getBufferUsage(), is(0f));
    assertThat(buffer.getAllocatedSize(), is(0L));
    assertThat(buffer.getBufferedDataSize(), is(0L));
    int totalLoopCount = concurrency * loopCount;
    int recordCount = 0;
    ObjectMapper objectMapper = new ObjectMapper(new MessagePackFactory());
    objectMapper.disable(JsonParser.Feature.AUTO_CLOSE_SOURCE);
    for (MockTCPSender.Event event : sender.getEvents()) {
        byte[] bytes = event.getAllBytes();
        MessageUnpacker messageUnpacker = MessagePack.newDefaultUnpacker(bytes);
        assertEquals(3, messageUnpacker.unpackArrayHeader());
        String tag = messageUnpacker.unpackString();
        byte[] payload = messageUnpacker.readPayload(messageUnpacker.unpackBinaryHeader());
        messageUnpacker = MessagePack.newDefaultUnpacker(payload);
        while (messageUnpacker.hasNext()) {
            assertEquals(2, messageUnpacker.unpackArrayHeader());
            ImmutableValue timestamp = messageUnpacker.unpackValue();
            int size = messageUnpacker.unpackMapHeader();
            assertEquals(3, size);
            Map<String, Object> data = new HashMap<String, Object>();
            for (int i = 0; i < size; i++) {
                String key = messageUnpacker.unpackString();
                ImmutableValue value = messageUnpacker.unpackValue();
                if (value.isStringValue()) {
                    data.put(key, value.asStringValue().asString());
                } else if (value.isIntegerValue()) {
                    data.put(key, value.asIntegerValue().asInt());
                }
            }
            analyzeResult(tag, timestamp, data, start, end, eventTime);
            recordCount++;
        }
    }
    assertEquals(totalLoopCount, recordCount);
    if (multiTags) {
        assertEquals(4, tagCounts.size());
        for (int i = 0; i < 4; i++) {
            int count = tagCounts.get(String.format("foodb%d.bartbl%d", i, i));
            assertTrue(totalLoopCount / 4 - 4 <= count && count <= totalLoopCount / 4 + 4);
        }
    } else {
        assertEquals(1, tagCounts.size());
        int count = tagCounts.get("foodb.bartbl");
        assertEquals(totalLoopCount, count);
    }
    assertEquals("komamitsu000000", minName);
    assertEquals(String.format("komamitsu%06d", loopCount - 1), maxName);
    assertEquals(0, minAge);
    assertEquals(loopCount - 1, maxAge);
    assertTrue(totalLoopCount / 31 - 5 <= longCommentCount && longCommentCount <= totalLoopCount / 31 + 5);
}