Examples with MessagePackFactory org.msgpack.jackson.dataformat.MessagePackFactory used on opensource projects

Example 1 with MessagePackFactory

use of org.msgpack.jackson.dataformat.MessagePackFactory in project fluency by komamitsu.

the class FluencyTestWithMockServer method testFluencyBase.

private void testFluencyBase(final FluencyFactory fluencyFactory, final Options options) throws Exception {
    LOG.info("testFluencyBase starts: options={}", options);
    final ArrayList<Integer> localPorts = new ArrayList<Integer>();
    final MockFluentdServer fluentd = new MockFluentdServer(options.sslEnabled);
    fluentd.start();
    final MockFluentdServer secondaryFluentd = new MockFluentdServer(options.sslEnabled, fluentd);
    secondaryFluentd.start();
    localPorts.add(fluentd.getLocalPort());
    localPorts.add(secondaryFluentd.getLocalPort());
    final AtomicReference<Fluency> fluency = new AtomicReference<Fluency>(fluencyFactory.generate(localPorts));
    if (options.fileBackup) {
        fluency.get().clearBackupFiles();
    }
    final ObjectMapper objectMapper = new ObjectMapper(new MessagePackFactory());
    final int maxNameLen = 200;
    final HashMap<Integer, String> nameLenTable = new HashMap<Integer, String>(maxNameLen);
    for (int i = 1; i <= maxNameLen; i++) {
        StringBuilder stringBuilder = new StringBuilder();
        for (int j = 0; j < i; j++) {
            stringBuilder.append('x');
        }
        nameLenTable.put(i, stringBuilder.toString());
    }
    final AtomicLong ageEventsSum = new AtomicLong();
    final AtomicLong nameEventsLength = new AtomicLong();
    final AtomicLong tag0EventsCounter = new AtomicLong();
    final AtomicLong tag1EventsCounter = new AtomicLong();
    final AtomicLong tag2EventsCounter = new AtomicLong();
    final AtomicLong tag3EventsCounter = new AtomicLong();
    try {
        final Random random = new Random();
        int concurrency = 10;
        final int reqNum = 6000;
        long start = System.currentTimeMillis();
        final CountDownLatch latch = new CountDownLatch(concurrency);
        final AtomicBoolean shouldFailOver = new AtomicBoolean(true);
        final AtomicBoolean shouldStopFluentd = new AtomicBoolean(true);
        final AtomicBoolean shouldStopFluency = new AtomicBoolean(true);
        final CountDownLatch fluentdCloseWaitLatch = new CountDownLatch(concurrency);
        final CountDownLatch fluencyCloseWaitLatch = new CountDownLatch(concurrency);
        ExecutorService es = Executors.newCachedThreadPool();
        for (int i = 0; i < concurrency; i++) {
            es.execute(new Runnable() {

                @Override
                public void run() {
                    for (int i = 0; i < reqNum; i++) {
                        if (Thread.currentThread().isInterrupted()) {
                            LOG.info("Interrupted...");
                            break;
                        }
                        if (options.failover) {
                            if (i == reqNum / 2) {
                                if (shouldFailOver.getAndSet(false)) {
                                    LOG.info("Failing over...");
                                    try {
                                        secondaryFluentd.stop();
                                    } catch (IOException e) {
                                        LOG.warn("Failed to stop secondary fluentd", e);
                                    }
                                }
                            }
                        } else if (options.fileBackup) {
                            if (i == reqNum / 2) {
                                if (shouldStopFluentd.getAndSet(false)) {
                                    LOG.info("Stopping Fluentd...");
                                    try {
                                        fluentd.stop();
                                        secondaryFluentd.stop();
                                    } catch (IOException e) {
                                        LOG.warn("Failed to stop Fluentd", e);
                                    }
                                }
                                fluentdCloseWaitLatch.countDown();
                                try {
                                    assertTrue(fluentdCloseWaitLatch.await(20, TimeUnit.SECONDS));
                                } catch (InterruptedException e) {
                                    LOG.warn("Interrupted", e);
                                }
                                if (shouldStopFluency.getAndSet(false)) {
                                    LOG.info("Stopping Fluency...");
                                    try {
                                        fluency.get().close();
                                        TimeUnit.SECONDS.sleep(2);
                                    } catch (Exception e) {
                                        LOG.warn("Failed to stop Fluency", e);
                                    }
                                    LOG.info("Restarting Fluentd...");
                                    try {
                                        fluentd.start();
                                        secondaryFluentd.start();
                                        LOG.info("Restarting Fluency...");
                                        fluency.set(fluencyFactory.generate(Arrays.asList(fluentd.getLocalPort(), secondaryFluentd.getLocalPort())));
                                        TimeUnit.SECONDS.sleep(2);
                                    } catch (Exception e) {
                                        LOG.warn("Failed to restart Fluentd", e);
                                    }
                                }
                                fluencyCloseWaitLatch.countDown();
                                try {
                                    assertTrue(fluencyCloseWaitLatch.await(20, TimeUnit.SECONDS));
                                } catch (InterruptedException e) {
                                    LOG.warn("Interrupted", e);
                                }
                            }
                        }
                        int tagNum = i % 4;
                        final String tag = String.format("foodb%d.bartbl%d", tagNum, tagNum);
                        switch(tagNum) {
                            case 0:
                                tag0EventsCounter.incrementAndGet();
                                break;
                            case 1:
                                tag1EventsCounter.incrementAndGet();
                                break;
                            case 2:
                                tag2EventsCounter.incrementAndGet();
                                break;
                            case 3:
                                tag3EventsCounter.incrementAndGet();
                                break;
                            default:
                                throw new RuntimeException("Never reach here");
                        }
                        int rand = random.nextInt(maxNameLen);
                        final Map<String, Object> hashMap = new HashMap<String, Object>();
                        String name = nameLenTable.get(rand + 1);
                        nameEventsLength.addAndGet(name.length());
                        hashMap.put("name", name);
                        rand = random.nextInt(100);
                        int age = rand;
                        ageEventsSum.addAndGet(age);
                        hashMap.put("age", age);
                        hashMap.put("comment", "hello, world");
                        hashMap.put("rate", 1.23);
                        try {
                            Exception exception = null;
                            for (int retry = 0; retry < 10; retry++) {
                                try {
                                    switch(options.emitType) {
                                        case MAP:
                                            {
                                                fluency.get().emit(tag, hashMap);
                                                break;
                                            }
                                        case MAP_WITH_EVENT_TIME:
                                            {
                                                EventTime eventTime = EventTime.fromEpochMilli(System.currentTimeMillis());
                                                fluency.get().emit(tag, eventTime, hashMap);
                                                break;
                                            }
                                        case MSGPACK_MAP_VALUE_BYTES:
                                            {
                                                byte[] bytes = objectMapper.writeValueAsBytes(hashMap);
                                                fluency.get().emit(tag, bytes, 0, bytes.length);
                                                break;
                                            }
                                        case MSGPACK_MAP_VALUE_BYTES_WITH_EVENT_TIME:
                                            {
                                                EventTime eventTime = EventTime.fromEpochMilli(System.currentTimeMillis());
                                                byte[] bytes = objectMapper.writeValueAsBytes(hashMap);
                                                fluency.get().emit(tag, eventTime, bytes, 0, bytes.length);
                                                break;
                                            }
                                        case MSGPACK_MAP_VALUE_BYTEBUFFER:
                                            {
                                                byte[] bytes = objectMapper.writeValueAsBytes(hashMap);
                                                fluency.get().emit(tag, ByteBuffer.wrap(bytes));
                                                break;
                                            }
                                        case MSGPACK_MAP_VALUE_BYTEBUFFER_WITH_EVENT_TIME:
                                            {
                                                EventTime eventTime = EventTime.fromEpochMilli(System.currentTimeMillis());
                                                byte[] bytes = objectMapper.writeValueAsBytes(hashMap);
                                                fluency.get().emit(tag, eventTime, ByteBuffer.wrap(bytes));
                                                break;
                                            }
                                    }
                                    exception = null;
                                    break;
                                } catch (Exception e) {
                                    exception = e;
                                    try {
                                        TimeUnit.SECONDS.sleep(1);
                                    } catch (InterruptedException e1) {
                                    }
                                }
                            }
                            if (exception != null) {
                                throw exception;
                            }
                        } catch (Exception e) {
                            LOG.warn("Exception occurred", e);
                        }
                    }
                    latch.countDown();
                }
            });
        }
        if (!latch.await(60, TimeUnit.SECONDS)) {
            assertTrue("Sending all requests is timed out", false);
        }
        if (options.closeInsteadOfFlush) {
            fluency.get().close();
        } else {
            fluency.get().flush();
            fluency.get().waitUntilAllBufferFlushed(20);
        }
        fluentd.waitUntilEventsStop();
        fluentd.stop();
        secondaryFluentd.waitUntilEventsStop();
        secondaryFluentd.stop();
        if (options.failover) {
            assertThat(fluentd.connectCounter.get(), is(greaterThan(0L)));
            assertThat(fluentd.connectCounter.get(), is(lessThanOrEqualTo(10L)));
            assertThat(fluentd.closeCounter.get(), is(greaterThan(0L)));
            assertThat(fluentd.closeCounter.get(), is(lessThanOrEqualTo(10L)));
        } else {
            assertThat(fluentd.connectCounter.get(), is(greaterThan(0L)));
            assertThat(fluentd.connectCounter.get(), is(lessThanOrEqualTo(2L)));
            if (options.closeInsteadOfFlush) {
                assertThat(fluentd.closeCounter.get(), is(greaterThan(0L)));
            } else {
                assertThat(fluentd.closeCounter.get(), is(0L));
            }
            assertThat(fluentd.closeCounter.get(), is(lessThanOrEqualTo(2L)));
        }
        assertEquals((long) concurrency * reqNum, fluentd.ageEventsCounter.get());
        assertEquals(ageEventsSum.get(), fluentd.ageEventsSum.get());
        assertEquals((long) concurrency * reqNum, fluentd.nameEventsCounter.get());
        assertEquals(nameEventsLength.get(), fluentd.nameEventsLength.get());
        assertEquals(tag0EventsCounter.get(), fluentd.tag0EventsCounter.get());
        assertEquals(tag1EventsCounter.get(), fluentd.tag1EventsCounter.get());
        assertEquals(tag2EventsCounter.get(), fluentd.tag2EventsCounter.get());
        assertEquals(tag3EventsCounter.get(), fluentd.tag3EventsCounter.get());
        System.out.println(System.currentTimeMillis() - start);
    } finally {
        fluency.get().close();
        fluentd.stop();
        secondaryFluentd.stop();
    }
}

Example 2 with MessagePackFactory

use of org.msgpack.jackson.dataformat.MessagePackFactory in project fluency by komamitsu.

the class BufferTestHelper method baseTestMessageBuffer.

void baseTestMessageBuffer(final int loopCount, final boolean multiTags, final boolean syncFlush, final boolean eventTime, final Buffer buffer) throws IOException, InterruptedException {
    assertThat(buffer.getBufferUsage(), is(0f));
    assertThat(buffer.getAllocatedSize(), is(0L));
    assertThat(buffer.getBufferedDataSize(), is(0L));
    final int concurrency = 4;
    final CountDownLatch latch = new CountDownLatch(concurrency);
    final MockTCPSender sender = new MockTCPSender(24229);
    Runnable emitTask = new Runnable() {

        @Override
        public void run() {
            try {
                for (int i = 0; i < loopCount; i++) {
                    HashMap<String, Object> data = new HashMap<String, Object>();
                    data.put("name", String.format("komamitsu%06d", i));
                    data.put("age", i);
                    data.put("comment", i % 31 == 0 ? longStr : "hello");
                    String tag = multiTags ? String.format("foodb%d.bartbl%d", i % 4, i % 4) : "foodb.bartbl";
                    if (eventTime) {
                        buffer.append(tag, new EventTime((int) (System.currentTimeMillis() / 1000), 999999999), data);
                    } else {
                        buffer.append(tag, System.currentTimeMillis(), data);
                    }
                }
                latch.countDown();
            } catch (Exception e) {
                e.printStackTrace();
            }
        }
    };
    final ExecutorService flushService = Executors.newSingleThreadExecutor();
    if (!syncFlush) {
        flushService.execute(new Runnable() {

            @Override
            public void run() {
                while (!flushService.isShutdown()) {
                    try {
                        TimeUnit.MILLISECONDS.sleep(100L);
                        buffer.flush(sender, false);
                    } catch (InterruptedException e) {
                        Thread.currentThread().interrupt();
                    } catch (IOException e) {
                        e.printStackTrace();
                        return;
                    }
                }
            }
        });
    }
    long start = System.currentTimeMillis();
    final ExecutorService executorService = Executors.newFixedThreadPool(concurrency);
    for (int i = 0; i < concurrency; i++) {
        executorService.execute(emitTask);
    }
    assertTrue(latch.await(10, TimeUnit.SECONDS));
    assertThat(buffer.getBufferUsage(), is(greaterThan(0f)));
    assertThat(buffer.getAllocatedSize(), is(greaterThan(0L)));
    assertThat(buffer.getBufferedDataSize(), is(greaterThan(0L)));
    buffer.flush(sender, true);
    buffer.close();
    long end = System.currentTimeMillis();
    executorService.shutdown();
    executorService.awaitTermination(10, TimeUnit.SECONDS);
    if (!executorService.isTerminated()) {
        executorService.shutdownNow();
    }
    flushService.shutdown();
    flushService.awaitTermination(10, TimeUnit.SECONDS);
    if (!flushService.isTerminated()) {
        flushService.shutdownNow();
    }
    // Just in case
    buffer.close();
    assertThat(buffer.getBufferUsage(), is(0f));
    assertThat(buffer.getAllocatedSize(), is(0L));
    assertThat(buffer.getBufferedDataSize(), is(0L));
    int totalLoopCount = concurrency * loopCount;
    int recordCount = 0;
    ObjectMapper objectMapper = new ObjectMapper(new MessagePackFactory());
    objectMapper.disable(JsonParser.Feature.AUTO_CLOSE_SOURCE);
    for (MockTCPSender.Event event : sender.getEvents()) {
        byte[] bytes = event.getAllBytes();
        MessageUnpacker messageUnpacker = MessagePack.newDefaultUnpacker(bytes);
        assertEquals(3, messageUnpacker.unpackArrayHeader());
        String tag = messageUnpacker.unpackString();
        byte[] payload = messageUnpacker.readPayload(messageUnpacker.unpackBinaryHeader());
        messageUnpacker = MessagePack.newDefaultUnpacker(payload);
        while (messageUnpacker.hasNext()) {
            assertEquals(2, messageUnpacker.unpackArrayHeader());
            ImmutableValue timestamp = messageUnpacker.unpackValue();
            int size = messageUnpacker.unpackMapHeader();
            assertEquals(3, size);
            Map<String, Object> data = new HashMap<String, Object>();
            for (int i = 0; i < size; i++) {
                String key = messageUnpacker.unpackString();
                ImmutableValue value = messageUnpacker.unpackValue();
                if (value.isStringValue()) {
                    data.put(key, value.asStringValue().asString());
                } else if (value.isIntegerValue()) {
                    data.put(key, value.asIntegerValue().asInt());
                }
            }
            analyzeResult(tag, timestamp, data, start, end, eventTime);
            recordCount++;
        }
    }
    assertEquals(totalLoopCount, recordCount);
    if (multiTags) {
        assertEquals(4, tagCounts.size());
        for (int i = 0; i < 4; i++) {
            int count = tagCounts.get(String.format("foodb%d.bartbl%d", i, i));
            assertTrue(totalLoopCount / 4 - 4 <= count && count <= totalLoopCount / 4 + 4);
        }
    } else {
        assertEquals(1, tagCounts.size());
        int count = tagCounts.get("foodb.bartbl");
        assertEquals(totalLoopCount, count);
    }
    assertEquals("komamitsu000000", minName);
    assertEquals(String.format("komamitsu%06d", loopCount - 1), maxName);
    assertEquals(0, minAge);
    assertEquals(loopCount - 1, maxAge);
    assertTrue(totalLoopCount / 31 - 5 <= longCommentCount && longCommentCount <= totalLoopCount / 31 + 5);
}

Example 3 with MessagePackFactory

use of org.msgpack.jackson.dataformat.MessagePackFactory in project fluency by komamitsu.

the class EventTimeTest method serialize.

@Test
public void serialize() throws JsonProcessingException {
    long now = System.currentTimeMillis();
    EventTime eventTime = EventTime.fromEpoch((int) (now / 1000), 999999999);
    ObjectMapper objectMapper = new ObjectMapper(new MessagePackFactory());
    byte[] bytes = objectMapper.writeValueAsBytes(eventTime);
    ByteBuffer byteBuffer = ByteBuffer.wrap(bytes);
    assertThat(byteBuffer.get(), is((byte) 0xD7));
    assertThat(byteBuffer.get(), is((byte) 0x00));
    assertThat(byteBuffer.getInt(), is((int) (now / 1000)));
    assertThat(byteBuffer.getInt(), is(999999999));
}

Example 4 with MessagePackFactory

use of org.msgpack.jackson.dataformat.MessagePackFactory in project zeebe by zeebe-io.

the class PartitionsResponseTest method shouldEncodePartitions.

@Test
public void shouldEncodePartitions() throws Exception {
    // given
    final PartitionsResponse response = new PartitionsResponse();
    response.addPartition(1, BufferUtil.wrapString("default-topic"));
    response.addPartition(2, BufferUtil.wrapString("foo"));
    response.addPartition(3, BufferUtil.wrapString("foo"));
    // when
    final UnsafeBuffer buf = new UnsafeBuffer(new byte[response.getLength()]);
    response.write(buf, 0);
    // then
    final ObjectMapper objectMapper = new ObjectMapper(new MessagePackFactory());
    final JsonNode deserializedResponse = objectMapper.readTree(new DirectBufferInputStream(buf));
    assertThat(deserializedResponse.isObject()).isTrue();
    assertThat(deserializedResponse.fieldNames()).containsExactly("partitions");
    final JsonNode partitions = deserializedResponse.get("partitions");
    assertThat(partitions.isArray()).isTrue();
    assertThat(partitions.size()).isEqualTo(3);
    final JsonNode partition3 = partitions.get(0);
    assertThat(partition3.isObject()).isTrue();
    assertThat(partition3.get("topic").textValue()).isEqualTo("default-topic");
    assertThat(partition3.get("id").numberValue()).isEqualTo(1);
    final JsonNode partition1 = partitions.get(1);
    assertThat(partition1.isObject()).isTrue();
    assertThat(partition1.get("topic").textValue()).isEqualTo("foo");
    assertThat(partition1.get("id").numberValue()).isEqualTo(2);
    final JsonNode partition2 = partitions.get(2);
    assertThat(partition2.isObject()).isTrue();
    assertThat(partition2.get("topic").textValue()).isEqualTo("foo");
    assertThat(partition2.get("id").numberValue()).isEqualTo(3);
}