Examples with BinaryDecoder co.cask.cdap.common.io.BinaryDecoder used on opensource projects

Example 1 with BinaryDecoder

use of co.cask.cdap.common.io.BinaryDecoder in project cdap by caskdata.

the class QueueToStreamConsumer method poll.

@Override
public DequeueResult<StreamEvent> poll(int maxEvents, long timeout, TimeUnit timeoutUnit) throws IOException, InterruptedException {
    final DequeueResult<byte[]> result = consumer.dequeue(maxEvents);
    // Decode byte array into stream event
    ImmutableList.Builder<StreamEvent> builder = ImmutableList.builder();
    for (byte[] content : result) {
        try {
            builder.add(STREAM_EVENT_CODEC.decodePayload(content));
        } catch (Throwable t) {
            // If failed to decode, it maybe using old (pre 2.1) stream codec. Try to decode with old one.
            ByteBuffer buffer = ByteBuffer.wrap(content);
            SchemaHash schemaHash = new SchemaHash(buffer);
            Preconditions.checkArgument(schemaHash.equals(StreamEventDataCodec.STREAM_DATA_SCHEMA.getSchemaHash()), "Schema from payload not matching with StreamEventData schema.");
            Decoder decoder = new BinaryDecoder(new ByteBufferInputStream(buffer));
            // In old schema, timestamp is not recorded.
            builder.add(new StreamEvent(StreamEventDataCodec.decode(decoder), 0));
        }
    }
    final List<StreamEvent> events = builder.build();
    return new DequeueResult<StreamEvent>() {

        @Override
        public boolean isEmpty() {
            return events.isEmpty();
        }

        @Override
        public void reclaim() {
            result.reclaim();
        }

        @Override
        public int size() {
            return events.size();
        }

        @Override
        public Iterator<StreamEvent> iterator() {
            return events.iterator();
        }
    };
}

Example 2 with BinaryDecoder

use of co.cask.cdap.common.io.BinaryDecoder in project cdap by caskdata.

the class StreamEventCodecTest method testEncodeDecodeWithDatumDecoder.

@Test
public void testEncodeDecodeWithDatumDecoder() throws UnsupportedTypeException, IOException {
    StreamEvent event = new StreamEvent(Maps.<String, String>newHashMap(), ByteBuffer.wrap("Event string".getBytes(Charsets.UTF_8)));
    StreamEventCodec codec = new StreamEventCodec();
    ByteBuffer payload = ByteBuffer.wrap(codec.encodePayload(event));
    SchemaHash schemaHash = new SchemaHash(payload);
    Schema schema = new ReflectionSchemaGenerator().generate(StreamEvent.class);
    Assert.assertEquals(schema.getSchemaHash(), schemaHash);
    StreamEvent decoded = new ReflectionDatumReader<>(schema, TypeToken.of(StreamEvent.class)).read(new BinaryDecoder(new ByteBufferInputStream(payload)), schema);
    Assert.assertEquals(event.getHeaders(), decoded.getHeaders());
    Assert.assertEquals(event.getBody(), decoded.getBody());
}

Example 3 with BinaryDecoder

use of co.cask.cdap.common.io.BinaryDecoder in project cdap by caskdata.

the class RecordWithString method testReduceProjection.

@Test
public void testReduceProjection() throws IOException, UnsupportedTypeException {
    PipedOutputStream output = new PipedOutputStream();
    PipedInputStream input = new PipedInputStream(output);
    Schema sourceSchema = new ReflectionSchemaGenerator().generate(MoreFields.class);
    Schema targetSchema = new ReflectionSchemaGenerator().generate(LessFields.class);
    MoreFields moreFields = new MoreFields(10, 20.2, "30", ImmutableList.of("1", "2"));
    new ReflectionDatumWriter<MoreFields>(sourceSchema).encode(moreFields, new BinaryEncoder(output));
    LessFields lessFields = new ReflectionDatumReader<>(targetSchema, TypeToken.of(LessFields.class)).read(new BinaryDecoder(input), sourceSchema);
    Assert.assertEquals("30", lessFields.k);
    Assert.assertEquals(moreFields.inner.b, lessFields.inner.b);
}

Example 4 with BinaryDecoder

use of co.cask.cdap.common.io.BinaryDecoder in project cdap by caskdata.

the class RecordWithString method testEmptyValue.

// this tests that the datum reader treats empty fields correctly. It reproduces the issue in ENG-2404.
@Test
public void testEmptyValue() throws UnsupportedTypeException, IOException {
    Schema schema = new ReflectionSchemaGenerator().generate(RecordWithString.class);
    TypeRepresentation typeRep = new TypeRepresentation(RecordWithString.class);
    DatumWriter<RecordWithString> datumWriter = new ReflectionDatumWriter<>(schema);
    @SuppressWarnings("unchecked") ReflectionDatumReader<RecordWithString> datumReader = new ReflectionDatumReader<>(schema, (TypeToken<RecordWithString>) TypeToken.of(typeRep.toType()));
    RecordWithString record = new RecordWithString();
    record.setA(42);
    record.setTheString("");
    ByteArrayOutputStream bos = new ByteArrayOutputStream();
    BinaryEncoder encoder = new BinaryEncoder(bos);
    datumWriter.encode(record, encoder);
    byte[] bytes = bos.toByteArray();
    ByteArrayInputStream bis = new ByteArrayInputStream(bytes);
    BinaryDecoder decoder = new BinaryDecoder(bis);
    RecordWithString rec = datumReader.read(decoder, schema);
    Assert.assertEquals(record.getA(), rec.getA());
    Assert.assertEquals(record.getTheString(), rec.getTheString());
}

Example 5 with BinaryDecoder

use of co.cask.cdap.common.io.BinaryDecoder in project cdap by caskdata.

the class ASMDatumCodecTest method testMap.

@Test
public void testMap() throws IOException, UnsupportedTypeException {
    TypeToken<Map<String, List<String>>> type = new TypeToken<Map<String, List<String>>>() {
    };
    PipedOutputStream os = new PipedOutputStream();
    PipedInputStream is = new PipedInputStream(os);
    DatumWriter<Map<String, List<String>>> writer = getWriter(type);
    ImmutableMap<String, List<String>> map = ImmutableMap.<String, List<String>>of("k1", Lists.newArrayList("v1"), "k2", Lists.newArrayList("v2", null));
    writer.encode(map, new BinaryEncoder(os));
    ReflectionDatumReader<Map<String, List<String>>> reader = new ReflectionDatumReader<>(getSchema(type), type);
    Assert.assertEquals(map, reader.read(new BinaryDecoder(is), getSchema(type)));
}