Search in sources :

Example 16 with BinaryEncoder

use of co.cask.cdap.common.io.BinaryEncoder in project cdap by caskdata.

the class DatumOutputEmitter method emit.

@Override
public void emit(T data, Map<String, Object> partitions) {
    try {
        ByteArrayOutputStream output = new ByteArrayOutputStream();
        output.write(schemaHash);
        writer.encode(data, new BinaryEncoder(output));
        producerSupplier.get().enqueue(new QueueEntry(Maps.transformValues(partitions, PARTITION_MAP_TRANSFORMER), output.toByteArray()));
    } catch (IOException e) {
        throw Throwables.propagate(e);
    }
}
Also used : BinaryEncoder(co.cask.cdap.common.io.BinaryEncoder) ByteArrayOutputStream(java.io.ByteArrayOutputStream) IOException(java.io.IOException) QueueEntry(co.cask.cdap.data2.queue.QueueEntry)

Example 17 with BinaryEncoder

use of co.cask.cdap.common.io.BinaryEncoder in project cdap by caskdata.

the class QueueEntry method serializeHashKeys.

public static byte[] serializeHashKeys(Map<String, Integer> hashKeys) throws IOException {
    // many entries will have no hash keys. Reuse a static value for that
    if (hashKeys == null || hashKeys.isEmpty()) {
        return SERIALIZED_EMPTY_HASH_KEYS;
    }
    ByteArrayOutputStream bos = new ByteArrayOutputStream();
    Encoder encoder = new BinaryEncoder(bos);
    encoder.writeInt(hashKeys.size());
    for (Map.Entry<String, Integer> entry : hashKeys.entrySet()) {
        encoder.writeString(entry.getKey()).writeInt(entry.getValue());
    }
    // per Avro spec, end with a (block of length) zero
    encoder.writeInt(0);
    return bos.toByteArray();
}
Also used : BinaryEncoder(co.cask.cdap.common.io.BinaryEncoder) BinaryEncoder(co.cask.cdap.common.io.BinaryEncoder) Encoder(co.cask.cdap.common.io.Encoder) ByteArrayOutputStream(java.io.ByteArrayOutputStream) ImmutableMap(com.google.common.collect.ImmutableMap) Map(java.util.Map)

Example 18 with BinaryEncoder

use of co.cask.cdap.common.io.BinaryEncoder in project cdap by caskdata.

the class CodecTest method testCodec.

@Test
public void testCodec() throws IOException {
    PipedOutputStream output = new PipedOutputStream();
    PipedInputStream input = new PipedInputStream(output);
    Encoder encoder = new BinaryEncoder(output);
    Decoder decoder = new BinaryDecoder(input);
    encoder.writeNull();
    Assert.assertNull(decoder.readNull());
    encoder.writeBool(true);
    Assert.assertTrue(decoder.readBool());
    encoder.writeBool(false);
    Assert.assertFalse(decoder.readBool());
    encoder.writeInt(0);
    Assert.assertEquals(0, decoder.readInt());
    encoder.writeInt(-1);
    Assert.assertEquals(-1, decoder.readInt());
    encoder.writeInt(1234);
    Assert.assertEquals(1234, decoder.readInt());
    encoder.writeInt(-1234);
    Assert.assertEquals(-1234, decoder.readInt());
    encoder.writeInt(Short.MAX_VALUE);
    Assert.assertEquals(Short.MAX_VALUE, decoder.readInt());
    encoder.writeInt(Short.MIN_VALUE);
    Assert.assertEquals(Short.MIN_VALUE, decoder.readInt());
    encoder.writeInt(Integer.MAX_VALUE);
    Assert.assertEquals(Integer.MAX_VALUE, decoder.readInt());
    encoder.writeInt(Integer.MIN_VALUE);
    Assert.assertEquals(Integer.MIN_VALUE, decoder.readInt());
    encoder.writeLong(0);
    Assert.assertEquals(0, decoder.readLong());
    encoder.writeLong(-20);
    Assert.assertEquals(-20, decoder.readLong());
    encoder.writeLong(30000);
    Assert.assertEquals(30000, decoder.readLong());
    encoder.writeLong(-600000);
    Assert.assertEquals(-600000, decoder.readLong());
    encoder.writeLong(Integer.MAX_VALUE);
    Assert.assertEquals(Integer.MAX_VALUE, decoder.readLong());
    encoder.writeLong(Integer.MIN_VALUE);
    Assert.assertEquals(Integer.MIN_VALUE, decoder.readLong());
    encoder.writeLong(Long.MAX_VALUE);
    Assert.assertEquals(Long.MAX_VALUE, decoder.readLong());
    encoder.writeLong(Long.MIN_VALUE);
    Assert.assertEquals(Long.MIN_VALUE, decoder.readLong());
    encoder.writeFloat(3.14f);
    Assert.assertEquals(3.14f, decoder.readFloat(), 0.0000001f);
    encoder.writeFloat(Short.MAX_VALUE);
    Assert.assertEquals(Short.MAX_VALUE, decoder.readFloat(), 0.0000001f);
    encoder.writeFloat(Integer.MIN_VALUE);
    Assert.assertEquals(Integer.MIN_VALUE, decoder.readFloat(), 0.0000001f);
    encoder.writeFloat((long) Integer.MAX_VALUE * Short.MAX_VALUE);
    Assert.assertEquals((long) Integer.MAX_VALUE * Short.MAX_VALUE, decoder.readFloat(), 0.0000001f);
    encoder.writeFloat(Float.MAX_VALUE);
    Assert.assertEquals(Float.MAX_VALUE, decoder.readFloat(), 0.0000001f);
    encoder.writeFloat(Float.MIN_VALUE);
    Assert.assertEquals(Float.MIN_VALUE, decoder.readFloat(), 0.0000001f);
    encoder.writeDouble(Math.E);
    Assert.assertEquals(Math.E, decoder.readDouble(), 0.0000001f);
    encoder.writeDouble(Integer.MAX_VALUE);
    Assert.assertEquals(Integer.MAX_VALUE, decoder.readDouble(), 0.0000001f);
    encoder.writeDouble(Long.MIN_VALUE);
    Assert.assertEquals(Long.MIN_VALUE, decoder.readDouble(), 0.0000001f);
    encoder.writeDouble((long) Integer.MAX_VALUE * Short.MAX_VALUE);
    Assert.assertEquals((long) Integer.MAX_VALUE * Short.MAX_VALUE, decoder.readDouble(), 0.0000001f);
    encoder.writeDouble(Double.MAX_VALUE);
    Assert.assertEquals(Double.MAX_VALUE, decoder.readDouble(), 0.0000001f);
    encoder.writeDouble(Double.MIN_VALUE);
    Assert.assertEquals(Double.MIN_VALUE, decoder.readDouble(), 0.0000001f);
    encoder.writeString("This is a testing message");
    Assert.assertEquals("This is a testing message", decoder.readString());
    String str = Character.toString((char) 200) + Character.toString((char) 20000) + Character.toString((char) 40000);
    encoder.writeString(str);
    Assert.assertEquals(str, decoder.readString());
    ByteBuffer buf = ByteBuffer.allocate(12);
    buf.asIntBuffer().put(10).put(1024).put(9999999);
    encoder.writeBytes(buf);
    IntBuffer inBuf = decoder.readBytes().asIntBuffer();
    Assert.assertEquals(10, inBuf.get());
    Assert.assertEquals(1024, inBuf.get());
    Assert.assertEquals(9999999, inBuf.get());
}
Also used : BinaryEncoder(co.cask.cdap.common.io.BinaryEncoder) BinaryEncoder(co.cask.cdap.common.io.BinaryEncoder) Encoder(co.cask.cdap.common.io.Encoder) IntBuffer(java.nio.IntBuffer) PipedOutputStream(java.io.PipedOutputStream) PipedInputStream(java.io.PipedInputStream) BinaryDecoder(co.cask.cdap.common.io.BinaryDecoder) Decoder(co.cask.cdap.common.io.Decoder) ByteBuffer(java.nio.ByteBuffer) BinaryDecoder(co.cask.cdap.common.io.BinaryDecoder) Test(org.junit.Test)

Example 19 with BinaryEncoder

use of co.cask.cdap.common.io.BinaryEncoder in project cdap by caskdata.

the class RecordWithString method testCollection.

@Test
public void testCollection() throws UnsupportedTypeException, IOException {
    List<String> list = Lists.newArrayList("1", "2", "3");
    Schema sourceSchema = new ReflectionSchemaGenerator().generate(new TypeToken<List<String>>() {
    }.getType());
    Schema targetSchema = new ReflectionSchemaGenerator().generate(new TypeToken<Set<String>>() {
    }.getType());
    PipedOutputStream output = new PipedOutputStream();
    PipedInputStream input = new PipedInputStream(output);
    new ReflectionDatumWriter<List<String>>(sourceSchema).encode(list, new BinaryEncoder(output));
    Set<String> set = new ReflectionDatumReader<>(targetSchema, new TypeToken<Set<String>>() {
    }).read(new BinaryDecoder(input), sourceSchema);
    Assert.assertEquals(Sets.newHashSet("1", "2", "3"), set);
    targetSchema = new ReflectionSchemaGenerator().generate(String[].class);
    new ReflectionDatumWriter<List<String>>(sourceSchema).encode(list, new BinaryEncoder(output));
    String[] array = new ReflectionDatumReader<>(targetSchema, new TypeToken<String[]>() {
    }).read(new BinaryDecoder(input), sourceSchema);
    Assert.assertArrayEquals(new String[] { "1", "2", "3" }, array);
}
Also used : Schema(co.cask.cdap.api.data.schema.Schema) PipedOutputStream(java.io.PipedOutputStream) PipedInputStream(java.io.PipedInputStream) ReflectionSchemaGenerator(co.cask.cdap.internal.io.ReflectionSchemaGenerator) BinaryDecoder(co.cask.cdap.common.io.BinaryDecoder) BinaryEncoder(co.cask.cdap.common.io.BinaryEncoder) TypeToken(com.google.common.reflect.TypeToken) ImmutableList(com.google.common.collect.ImmutableList) List(java.util.List) Test(org.junit.Test)

Example 20 with BinaryEncoder

use of co.cask.cdap.common.io.BinaryEncoder in project cdap by caskdata.

the class RecordWithString method testCircularRef.

@Test(expected = IOException.class)
public void testCircularRef() throws UnsupportedTypeException, IOException {
    Schema schema = new ReflectionSchemaGenerator().generate(Node.class);
    Node head = new Node();
    head.next = new Node();
    head.next.next = head;
    ByteArrayOutputStream output = new ByteArrayOutputStream();
    new ReflectionDatumWriter<Node>(schema).encode(head, new BinaryEncoder(output));
}
Also used : BinaryEncoder(co.cask.cdap.common.io.BinaryEncoder) Schema(co.cask.cdap.api.data.schema.Schema) ByteArrayOutputStream(java.io.ByteArrayOutputStream) ReflectionSchemaGenerator(co.cask.cdap.internal.io.ReflectionSchemaGenerator) Test(org.junit.Test)

Aggregations

BinaryEncoder (co.cask.cdap.common.io.BinaryEncoder)32 Test (org.junit.Test)24 BinaryDecoder (co.cask.cdap.common.io.BinaryDecoder)22 PipedInputStream (java.io.PipedInputStream)21 PipedOutputStream (java.io.PipedOutputStream)21 ReflectionDatumReader (co.cask.cdap.internal.io.ReflectionDatumReader)18 TypeToken (com.google.common.reflect.TypeToken)18 ByteArrayOutputStream (java.io.ByteArrayOutputStream)11 Encoder (co.cask.cdap.common.io.Encoder)7 Schema (co.cask.cdap.api.data.schema.Schema)6 ReflectionSchemaGenerator (co.cask.cdap.internal.io.ReflectionSchemaGenerator)6 ImmutableList (com.google.common.collect.ImmutableList)5 List (java.util.List)5 IOException (java.io.IOException)4 ReflectionDatumWriter (co.cask.cdap.internal.io.ReflectionDatumWriter)3 ImmutableMap (com.google.common.collect.ImmutableMap)3 Map (java.util.Map)3 ByteBuffer (java.nio.ByteBuffer)2 DataSetException (co.cask.cdap.api.dataset.DataSetException)1 StreamEvent (co.cask.cdap.api.flow.flowlet.StreamEvent)1