Search in sources :

Example 1 with BinaryEncoder

use of co.cask.cdap.common.io.BinaryEncoder in project cdap by caskdata.

the class RecordWithString method testReduceProjection.

@Test
public void testReduceProjection() throws IOException, UnsupportedTypeException {
    PipedOutputStream output = new PipedOutputStream();
    PipedInputStream input = new PipedInputStream(output);
    Schema sourceSchema = new ReflectionSchemaGenerator().generate(MoreFields.class);
    Schema targetSchema = new ReflectionSchemaGenerator().generate(LessFields.class);
    MoreFields moreFields = new MoreFields(10, 20.2, "30", ImmutableList.of("1", "2"));
    new ReflectionDatumWriter<MoreFields>(sourceSchema).encode(moreFields, new BinaryEncoder(output));
    LessFields lessFields = new ReflectionDatumReader<>(targetSchema, TypeToken.of(LessFields.class)).read(new BinaryDecoder(input), sourceSchema);
    Assert.assertEquals("30", lessFields.k);
    Assert.assertEquals(moreFields.inner.b, lessFields.inner.b);
}
Also used : BinaryEncoder(co.cask.cdap.common.io.BinaryEncoder) Schema(co.cask.cdap.api.data.schema.Schema) PipedOutputStream(java.io.PipedOutputStream) PipedInputStream(java.io.PipedInputStream) ReflectionSchemaGenerator(co.cask.cdap.internal.io.ReflectionSchemaGenerator) BinaryDecoder(co.cask.cdap.common.io.BinaryDecoder) Test(org.junit.Test)

Example 2 with BinaryEncoder

use of co.cask.cdap.common.io.BinaryEncoder in project cdap by caskdata.

the class RecordWithString method testEmptyValue.

// this tests that the datum reader treats empty fields correctly. It reproduces the issue in ENG-2404.
@Test
public void testEmptyValue() throws UnsupportedTypeException, IOException {
    Schema schema = new ReflectionSchemaGenerator().generate(RecordWithString.class);
    TypeRepresentation typeRep = new TypeRepresentation(RecordWithString.class);
    DatumWriter<RecordWithString> datumWriter = new ReflectionDatumWriter<>(schema);
    @SuppressWarnings("unchecked") ReflectionDatumReader<RecordWithString> datumReader = new ReflectionDatumReader<>(schema, (TypeToken<RecordWithString>) TypeToken.of(typeRep.toType()));
    RecordWithString record = new RecordWithString();
    record.setA(42);
    record.setTheString("");
    ByteArrayOutputStream bos = new ByteArrayOutputStream();
    BinaryEncoder encoder = new BinaryEncoder(bos);
    datumWriter.encode(record, encoder);
    byte[] bytes = bos.toByteArray();
    ByteArrayInputStream bis = new ByteArrayInputStream(bytes);
    BinaryDecoder decoder = new BinaryDecoder(bis);
    RecordWithString rec = datumReader.read(decoder, schema);
    Assert.assertEquals(record.getA(), rec.getA());
    Assert.assertEquals(record.getTheString(), rec.getTheString());
}
Also used : Schema(co.cask.cdap.api.data.schema.Schema) ReflectionDatumWriter(co.cask.cdap.internal.io.ReflectionDatumWriter) ReflectionDatumReader(co.cask.cdap.internal.io.ReflectionDatumReader) ByteArrayOutputStream(java.io.ByteArrayOutputStream) ReflectionSchemaGenerator(co.cask.cdap.internal.io.ReflectionSchemaGenerator) BinaryDecoder(co.cask.cdap.common.io.BinaryDecoder) BinaryEncoder(co.cask.cdap.common.io.BinaryEncoder) ByteArrayInputStream(java.io.ByteArrayInputStream) TypeRepresentation(co.cask.cdap.internal.io.TypeRepresentation) Test(org.junit.Test)

Example 3 with BinaryEncoder

use of co.cask.cdap.common.io.BinaryEncoder in project cdap by caskdata.

the class ASMDatumCodecTest method testSpeed.

@Ignore
@Test
public void testSpeed() throws UnsupportedTypeException, IOException {
    TypeToken<Node> type = new TypeToken<Node>() {
    };
    ByteArrayOutputStream os = new ByteArrayOutputStream(1024);
    long startTime;
    long endTime;
    Node writeValue = new Node((short) 1, new Node((short) 2, null, new Node((short) 3, null, null)), new Node((short) 4, new Node((short) 5, null, null), null));
    DatumWriter<Node> writer = getWriter(type);
    startTime = System.nanoTime();
    for (int i = 0; i < 100000; i++) {
        os.reset();
        writer.encode(writeValue, new BinaryEncoder(os));
    }
    endTime = System.nanoTime();
    System.out.println("Time spent: " + TimeUnit.MILLISECONDS.convert(endTime - startTime, TimeUnit.NANOSECONDS));
    ReflectionDatumWriter<Node> datumWriter = new ReflectionDatumWriter<>(getSchema(type));
    startTime = System.nanoTime();
    for (int i = 0; i < 100000; i++) {
        os.reset();
        datumWriter.encode(writeValue, new BinaryEncoder(os));
    }
    endTime = System.nanoTime();
    System.out.println("Time spent: " + TimeUnit.MILLISECONDS.convert(endTime - startTime, TimeUnit.NANOSECONDS));
    writer = getWriter(type);
    startTime = System.nanoTime();
    for (int i = 0; i < 100000; i++) {
        os.reset();
        writer.encode(writeValue, new BinaryEncoder(os));
    }
    endTime = System.nanoTime();
    System.out.println("Time spent: " + TimeUnit.MILLISECONDS.convert(endTime - startTime, TimeUnit.NANOSECONDS));
    datumWriter = new ReflectionDatumWriter<>(getSchema(type));
    startTime = System.nanoTime();
    for (int i = 0; i < 100000; i++) {
        os.reset();
        datumWriter.encode(writeValue, new BinaryEncoder(os));
    }
    endTime = System.nanoTime();
    System.out.println("Time spent: " + TimeUnit.MILLISECONDS.convert(endTime - startTime, TimeUnit.NANOSECONDS));
}
Also used : BinaryEncoder(co.cask.cdap.common.io.BinaryEncoder) TypeToken(com.google.common.reflect.TypeToken) ReflectionDatumWriter(co.cask.cdap.internal.io.ReflectionDatumWriter) ByteArrayOutputStream(java.io.ByteArrayOutputStream) Ignore(org.junit.Ignore) Test(org.junit.Test)

Example 4 with BinaryEncoder

use of co.cask.cdap.common.io.BinaryEncoder in project cdap by caskdata.

the class ASMDatumCodecTest method testMap.

@Test
public void testMap() throws IOException, UnsupportedTypeException {
    TypeToken<Map<String, List<String>>> type = new TypeToken<Map<String, List<String>>>() {
    };
    PipedOutputStream os = new PipedOutputStream();
    PipedInputStream is = new PipedInputStream(os);
    DatumWriter<Map<String, List<String>>> writer = getWriter(type);
    ImmutableMap<String, List<String>> map = ImmutableMap.<String, List<String>>of("k1", Lists.newArrayList("v1"), "k2", Lists.newArrayList("v2", null));
    writer.encode(map, new BinaryEncoder(os));
    ReflectionDatumReader<Map<String, List<String>>> reader = new ReflectionDatumReader<>(getSchema(type), type);
    Assert.assertEquals(map, reader.read(new BinaryDecoder(is), getSchema(type)));
}
Also used : PipedOutputStream(java.io.PipedOutputStream) ReflectionDatumReader(co.cask.cdap.internal.io.ReflectionDatumReader) PipedInputStream(java.io.PipedInputStream) BinaryDecoder(co.cask.cdap.common.io.BinaryDecoder) BinaryEncoder(co.cask.cdap.common.io.BinaryEncoder) TypeToken(com.google.common.reflect.TypeToken) ImmutableList(com.google.common.collect.ImmutableList) List(java.util.List) Map(java.util.Map) ImmutableMap(com.google.common.collect.ImmutableMap) Test(org.junit.Test)

Example 5 with BinaryEncoder

use of co.cask.cdap.common.io.BinaryEncoder in project cdap by caskdata.

the class ASMDatumCodecTest method testReferenceArray.

@Test
public void testReferenceArray() throws IOException, UnsupportedTypeException {
    TypeToken<String[]> type = new TypeToken<String[]>() {
    };
    PipedOutputStream os = new PipedOutputStream();
    PipedInputStream is = new PipedInputStream(os);
    String[] writeValue = new String[] { "1", "2", null, "3" };
    DatumWriter<String[]> writer = getWriter(type);
    writer.encode(writeValue, new BinaryEncoder(os));
    ReflectionDatumReader<String[]> reader = new ReflectionDatumReader<>(getSchema(type), type);
    String[] value = reader.read(new BinaryDecoder(is), getSchema(type));
    Assert.assertArrayEquals(writeValue, value);
}
Also used : BinaryEncoder(co.cask.cdap.common.io.BinaryEncoder) TypeToken(com.google.common.reflect.TypeToken) PipedOutputStream(java.io.PipedOutputStream) ReflectionDatumReader(co.cask.cdap.internal.io.ReflectionDatumReader) PipedInputStream(java.io.PipedInputStream) BinaryDecoder(co.cask.cdap.common.io.BinaryDecoder) Test(org.junit.Test)

Aggregations

BinaryEncoder (co.cask.cdap.common.io.BinaryEncoder)32 Test (org.junit.Test)24 BinaryDecoder (co.cask.cdap.common.io.BinaryDecoder)22 PipedInputStream (java.io.PipedInputStream)21 PipedOutputStream (java.io.PipedOutputStream)21 ReflectionDatumReader (co.cask.cdap.internal.io.ReflectionDatumReader)18 TypeToken (com.google.common.reflect.TypeToken)18 ByteArrayOutputStream (java.io.ByteArrayOutputStream)11 Encoder (co.cask.cdap.common.io.Encoder)7 Schema (co.cask.cdap.api.data.schema.Schema)6 ReflectionSchemaGenerator (co.cask.cdap.internal.io.ReflectionSchemaGenerator)6 ImmutableList (com.google.common.collect.ImmutableList)5 List (java.util.List)5 IOException (java.io.IOException)4 ReflectionDatumWriter (co.cask.cdap.internal.io.ReflectionDatumWriter)3 ImmutableMap (com.google.common.collect.ImmutableMap)3 Map (java.util.Map)3 ByteBuffer (java.nio.ByteBuffer)2 DataSetException (co.cask.cdap.api.dataset.DataSetException)1 StreamEvent (co.cask.cdap.api.flow.flowlet.StreamEvent)1