Examples with MessageUnpacker org.msgpack.core.MessageUnpacker used on opensource projects

Example 1 with MessageUnpacker

use of org.msgpack.core.MessageUnpacker in project td-client-java by treasure-data.

the class TestTDClient method submitJob.

@Test
public void submitJob() throws Exception {
    String jobId = client.submit(TDJobRequest.newPrestoQuery("sample_datasets", "-- td-client-java test\nselect count(*) cnt from nasdaq"));
    logger.debug("job id: " + jobId);
    int retryCount = 0;
    TDJobSummary tdJob = waitJobCompletion(jobId);
    TDJob jobInfo = client.jobInfo(jobId);
    logger.debug("job show result: " + tdJob);
    logger.debug("job info: " + jobInfo);
    Optional<String> schema = jobInfo.getResultSchema();
    assertTrue(schema.isPresent());
    assertEquals("[[\"cnt\", \"bigint\"]]", schema.get());
    JSONArray array = client.jobResult(jobId, TDResultFormat.JSON, new Function<InputStream, JSONArray>() {

        @Override
        public JSONArray apply(InputStream input) {
            try {
                String result = new String(ByteStreams.toByteArray(input), StandardCharsets.UTF_8);
                logger.info("result:\n" + result);
                return new JSONArray(result);
            } catch (Exception e) {
                throw Throwables.propagate(e);
            }
        }
    });
    assertEquals(1, array.length());
    assertEquals(8807278, array.getLong(0));
    // test msgpack.gz format
    client.jobResult(jobId, TDResultFormat.MESSAGE_PACK_GZ, new Function<InputStream, Object>() {

        @Override
        public Object apply(InputStream input) {
            try {
                logger.debug("Reading job result in msgpack.gz");
                MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(new GZIPInputStream(input));
                int rowCount = 0;
                while (unpacker.hasNext()) {
                    ArrayValue array = unpacker.unpackValue().asArrayValue();
                    assertEquals(1, array.size());
                    int numColumns = array.get(0).asIntegerValue().toInt();
                    assertEquals(8807278, numColumns);
                    rowCount++;
                }
                assertEquals(rowCount, 1);
                return null;
            } catch (IOException e) {
                throw Throwables.propagate(e);
            }
        }
    });
}

Example 2 with MessageUnpacker

use of org.msgpack.core.MessageUnpacker in project fluency by komamitsu.

the class FluencyTest method testWithoutAckResponse.

@Theory
public void testWithoutAckResponse(final boolean sslEnabled) throws Throwable {
    Exception exception = new ConfigurableTestServer(sslEnabled).run(new ConfigurableTestServer.WithClientSocket() {

        @Override
        public void run(Socket clientSocket) throws Exception {
            MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(clientSocket.getInputStream());
            assertEquals(3, unpacker.unpackArrayHeader());
            assertEquals("foo.bar", unpacker.unpackString());
            ImmutableRawValue rawValue = unpacker.unpackValue().asRawValue();
            Map<Value, Value> map = unpacker.unpackValue().asMapValue().map();
            assertEquals(1, map.size());
            assertEquals(rawValue.asByteArray().length, map.get(KEY_OPTION_SIZE).asIntegerValue().asInt());
            unpacker.close();
        }
    }, new ConfigurableTestServer.WithServerPort() {

        @Override
        public void run(int serverPort) throws Exception {
            Fluency fluency = Fluency.defaultFluency(serverPort, new Fluency.Config().setSslEnabled(sslEnabled));
            fluency.emit("foo.bar", new HashMap<String, Object>());
            fluency.close();
        }
    }, 5000);
    assertNull(exception);
}

Example 3 with MessageUnpacker

use of org.msgpack.core.MessageUnpacker in project fluency by komamitsu.

the class FluencyTest method testWithAckResponseWithProperToken.

@Theory
public void testWithAckResponseWithProperToken(final boolean sslEnabled) throws Throwable {
    Exception exception = new ConfigurableTestServer(sslEnabled).run(new ConfigurableTestServer.WithClientSocket() {

        @Override
        public void run(Socket clientSocket) throws Exception {
            MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(clientSocket.getInputStream());
            assertEquals(3, unpacker.unpackArrayHeader());
            assertEquals("foo.bar", unpacker.unpackString());
            ImmutableRawValue rawValue = unpacker.unpackValue().asRawValue();
            Map<Value, Value> map = unpacker.unpackValue().asMapValue().map();
            assertEquals(2, map.size());
            assertEquals(rawValue.asByteArray().length, map.get(KEY_OPTION_SIZE).asIntegerValue().asInt());
            String ackResponseToken = map.get(KEY_OPTION_CHUNK).asRawValue().asString();
            assertNotNull(ackResponseToken);
            MessagePacker packer = MessagePack.newDefaultPacker(clientSocket.getOutputStream());
            packer.packMapHeader(1).packString("ack").packString(ackResponseToken).close();
            // Close the input stream after closing the output stream to avoid closing a socket too early
            unpacker.close();
        }
    }, new ConfigurableTestServer.WithServerPort() {

        @Override
        public void run(int serverPort) throws Exception {
            Fluency fluency = Fluency.defaultFluency(serverPort, new Fluency.Config().setSslEnabled(sslEnabled).setAckResponseMode(true));
            fluency.emit("foo.bar", new HashMap<String, Object>());
            fluency.close();
        }
    }, 5000);
    assertNull(exception);
}

Example 4 with MessageUnpacker

use of org.msgpack.core.MessageUnpacker in project fluency by komamitsu.

the class FluencyTest method testWithAckResponseButNotReceiveToken.

@Theory
public void testWithAckResponseButNotReceiveToken(final boolean sslEnabled) throws Throwable {
    Exception exception = new ConfigurableTestServer(sslEnabled).run(new ConfigurableTestServer.WithClientSocket() {

        @Override
        public void run(Socket clientSocket) throws Exception {
            MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(clientSocket.getInputStream());
            assertEquals(3, unpacker.unpackArrayHeader());
            assertEquals("foo.bar", unpacker.unpackString());
            ImmutableRawValue rawValue = unpacker.unpackValue().asRawValue();
            Map<Value, Value> map = unpacker.unpackValue().asMapValue().map();
            assertEquals(2, map.size());
            assertEquals(rawValue.asByteArray().length, map.get(KEY_OPTION_SIZE).asIntegerValue().asInt());
            assertNotNull(map.get(KEY_OPTION_CHUNK).asRawValue().asString());
            unpacker.close();
        }
    }, new ConfigurableTestServer.WithServerPort() {

        @Override
        public void run(int serverPort) throws Exception {
            Fluency fluency = Fluency.defaultFluency(serverPort, new Fluency.Config().setSslEnabled(sslEnabled).setAckResponseMode(true));
            fluency.emit("foo.bar", new HashMap<String, Object>());
            fluency.close();
        }
    }, 5000);
    assertEquals(exception.getClass(), TimeoutException.class);
}

Example 5 with MessageUnpacker

use of org.msgpack.core.MessageUnpacker in project fluency by komamitsu.

the class BufferTestHelper method baseTestMessageBuffer.

void baseTestMessageBuffer(final int loopCount, final boolean multiTags, final boolean syncFlush, final boolean eventTime, final Buffer buffer) throws IOException, InterruptedException {
    assertThat(buffer.getBufferUsage(), is(0f));
    assertThat(buffer.getAllocatedSize(), is(0L));
    assertThat(buffer.getBufferedDataSize(), is(0L));
    final int concurrency = 4;
    final CountDownLatch latch = new CountDownLatch(concurrency);
    final MockTCPSender sender = new MockTCPSender(24229);
    Runnable emitTask = new Runnable() {

        @Override
        public void run() {
            try {
                for (int i = 0; i < loopCount; i++) {
                    HashMap<String, Object> data = new HashMap<String, Object>();
                    data.put("name", String.format("komamitsu%06d", i));
                    data.put("age", i);
                    data.put("comment", i % 31 == 0 ? longStr : "hello");
                    String tag = multiTags ? String.format("foodb%d.bartbl%d", i % 4, i % 4) : "foodb.bartbl";
                    if (eventTime) {
                        buffer.append(tag, new EventTime((int) (System.currentTimeMillis() / 1000), 999999999), data);
                    } else {
                        buffer.append(tag, System.currentTimeMillis(), data);
                    }
                }
                latch.countDown();
            } catch (Exception e) {
                e.printStackTrace();
            }
        }
    };
    final ExecutorService flushService = Executors.newSingleThreadExecutor();
    if (!syncFlush) {
        flushService.execute(new Runnable() {

            @Override
            public void run() {
                while (!flushService.isShutdown()) {
                    try {
                        TimeUnit.MILLISECONDS.sleep(100L);
                        buffer.flush(sender, false);
                    } catch (InterruptedException e) {
                        Thread.currentThread().interrupt();
                    } catch (IOException e) {
                        e.printStackTrace();
                        return;
                    }
                }
            }
        });
    }
    long start = System.currentTimeMillis();
    final ExecutorService executorService = Executors.newFixedThreadPool(concurrency);
    for (int i = 0; i < concurrency; i++) {
        executorService.execute(emitTask);
    }
    assertTrue(latch.await(10, TimeUnit.SECONDS));
    assertThat(buffer.getBufferUsage(), is(greaterThan(0f)));
    assertThat(buffer.getAllocatedSize(), is(greaterThan(0L)));
    assertThat(buffer.getBufferedDataSize(), is(greaterThan(0L)));
    buffer.flush(sender, true);
    buffer.close();
    long end = System.currentTimeMillis();
    executorService.shutdown();
    executorService.awaitTermination(10, TimeUnit.SECONDS);
    if (!executorService.isTerminated()) {
        executorService.shutdownNow();
    }
    flushService.shutdown();
    flushService.awaitTermination(10, TimeUnit.SECONDS);
    if (!flushService.isTerminated()) {
        flushService.shutdownNow();
    }
    // Just in case
    buffer.close();
    assertThat(buffer.getBufferUsage(), is(0f));
    assertThat(buffer.getAllocatedSize(), is(0L));
    assertThat(buffer.getBufferedDataSize(), is(0L));
    int totalLoopCount = concurrency * loopCount;
    int recordCount = 0;
    ObjectMapper objectMapper = new ObjectMapper(new MessagePackFactory());
    objectMapper.disable(JsonParser.Feature.AUTO_CLOSE_SOURCE);
    for (MockTCPSender.Event event : sender.getEvents()) {
        byte[] bytes = event.getAllBytes();
        MessageUnpacker messageUnpacker = MessagePack.newDefaultUnpacker(bytes);
        assertEquals(3, messageUnpacker.unpackArrayHeader());
        String tag = messageUnpacker.unpackString();
        byte[] payload = messageUnpacker.readPayload(messageUnpacker.unpackBinaryHeader());
        messageUnpacker = MessagePack.newDefaultUnpacker(payload);
        while (messageUnpacker.hasNext()) {
            assertEquals(2, messageUnpacker.unpackArrayHeader());
            ImmutableValue timestamp = messageUnpacker.unpackValue();
            int size = messageUnpacker.unpackMapHeader();
            assertEquals(3, size);
            Map<String, Object> data = new HashMap<String, Object>();
            for (int i = 0; i < size; i++) {
                String key = messageUnpacker.unpackString();
                ImmutableValue value = messageUnpacker.unpackValue();
                if (value.isStringValue()) {
                    data.put(key, value.asStringValue().asString());
                } else if (value.isIntegerValue()) {
                    data.put(key, value.asIntegerValue().asInt());
                }
            }
            analyzeResult(tag, timestamp, data, start, end, eventTime);
            recordCount++;
        }
    }
    assertEquals(totalLoopCount, recordCount);
    if (multiTags) {
        assertEquals(4, tagCounts.size());
        for (int i = 0; i < 4; i++) {
            int count = tagCounts.get(String.format("foodb%d.bartbl%d", i, i));
            assertTrue(totalLoopCount / 4 - 4 <= count && count <= totalLoopCount / 4 + 4);
        }
    } else {
        assertEquals(1, tagCounts.size());
        int count = tagCounts.get("foodb.bartbl");
        assertEquals(totalLoopCount, count);
    }
    assertEquals("komamitsu000000", minName);
    assertEquals(String.format("komamitsu%06d", loopCount - 1), maxName);
    assertEquals(0, minAge);
    assertEquals(loopCount - 1, maxAge);
    assertTrue(totalLoopCount / 31 - 5 <= longCommentCount && longCommentCount <= totalLoopCount / 31 + 5);
}