Examples with FormatSpecification io.cdap.cdap.api.data.format.FormatSpecification used on opensource projects

Example 16 with FormatSpecification

use of io.cdap.cdap.api.data.format.FormatSpecification in project cdap by caskdata.

the class AvroRecordFormatTest method testNestedRecord.

@Test
public void testNestedRecord() throws Exception {
    Schema innerSchema = Schema.recordOf("inner", Schema.Field.of("int", Schema.of(Schema.Type.INT)), Schema.Field.of("double", Schema.of(Schema.Type.DOUBLE)), Schema.Field.of("array", Schema.arrayOf(Schema.of(Schema.Type.FLOAT))), Schema.Field.of("map", Schema.mapOf(Schema.of(Schema.Type.STRING), Schema.of(Schema.Type.STRING))));
    Schema schema = Schema.recordOf("record", Schema.Field.of("int", Schema.of(Schema.Type.INT)), Schema.Field.of("record", innerSchema));
    org.apache.avro.Schema avroInnerSchema = convertSchema(innerSchema);
    org.apache.avro.Schema avroSchema = convertSchema(schema);
    GenericRecord record = new GenericRecordBuilder(avroSchema).set("int", Integer.MAX_VALUE).set("record", new GenericRecordBuilder(avroInnerSchema).set("int", 5).set("double", 3.14159).set("array", ImmutableList.of(1.0f, 2.0f)).set("map", ImmutableMap.of("key", "value")).build()).build();
    FormatSpecification formatSpecification = new FormatSpecification(Formats.AVRO, schema, Collections.emptyMap());
    RecordFormat<ByteBuffer, StructuredRecord> format = RecordFormats.createInitializedFormat(formatSpecification);
    StructuredRecord actual = format.read(toByteBuffer(record));
    Assert.assertEquals(Integer.MAX_VALUE, (int) actual.get("int"));
    StructuredRecord actualInner = actual.get("record");
    Assert.assertEquals(5, (int) actualInner.get("int"));
    Assert.assertEquals(3.14159d, actualInner.get("double"), 0.0001d);
    List<Float> array = actualInner.get("array");
    Assert.assertEquals(ImmutableList.of(1.0f, 2.0f), array);
    Map<String, String> map = actualInner.get("map");
    Assert.assertEquals(ImmutableMap.of("key", "value"), map);
}

Example 17 with FormatSpecification

use of io.cdap.cdap.api.data.format.FormatSpecification in project cdap by caskdata.

the class AvroRecordFormatTest method testMultipleReads.

@Test
public void testMultipleReads() throws Exception {
    Schema schema = Schema.recordOf("record", Schema.Field.of("x", Schema.of(Schema.Type.INT)));
    FormatSpecification formatSpecification = new FormatSpecification(Formats.AVRO, schema, Collections.emptyMap());
    org.apache.avro.Schema avroSchema = convertSchema(schema);
    RecordFormat<ByteBuffer, StructuredRecord> format = RecordFormats.createInitializedFormat(formatSpecification);
    GenericRecord record = new GenericRecordBuilder(avroSchema).set("x", 5).build();
    StructuredRecord actual = format.read(toByteBuffer(record));
    Assert.assertEquals(5, (int) actual.get("x"));
    record = new GenericRecordBuilder(avroSchema).set("x", 10).build();
    actual = format.read(toByteBuffer(record));
    Assert.assertEquals(10, (int) actual.get("x"));
}

Example 18 with FormatSpecification

use of io.cdap.cdap.api.data.format.FormatSpecification in project cdap by caskdata.

the class DelimitedStringsRecordFormatTest method testCSV.

@Test
public void testCSV() throws Exception {
    FormatSpecification spec = new FormatSpecification(Formats.CSV, null, Collections.<String, String>emptyMap());
    RecordFormat<ByteBuffer, StructuredRecord> format = RecordFormats.createInitializedFormat(spec);
    String body = "userX,actionY,itemZ";
    StructuredRecord output = format.read(ByteBuffer.wrap(Bytes.toBytes(body)));
    String[] actual = output.get("body");
    String[] expected = body.split(",");
    Assert.assertArrayEquals(expected, actual);
}

Example 19 with FormatSpecification

use of io.cdap.cdap.api.data.format.FormatSpecification in project cdap by caskdata.

the class DelimitedStringsRecordFormatTest method testNullableFieldsAllowedInSchema.

@Test
public void testNullableFieldsAllowedInSchema() throws UnsupportedTypeException {
    Schema schema = Schema.recordOf("event", Schema.Field.of("f1", Schema.unionOf(Schema.of(Schema.Type.BOOLEAN), Schema.of(Schema.Type.NULL))), Schema.Field.of("f2", Schema.unionOf(Schema.of(Schema.Type.INT), Schema.of(Schema.Type.NULL))), Schema.Field.of("f3", Schema.unionOf(Schema.of(Schema.Type.FLOAT), Schema.of(Schema.Type.NULL))), Schema.Field.of("f4", Schema.unionOf(Schema.of(Schema.Type.DOUBLE), Schema.of(Schema.Type.NULL))), Schema.Field.of("f5", Schema.unionOf(Schema.of(Schema.Type.BYTES), Schema.of(Schema.Type.NULL))), Schema.Field.of("f6", Schema.unionOf(Schema.of(Schema.Type.STRING), Schema.of(Schema.Type.NULL))));
    DelimitedStringsRecordFormat format = new DelimitedStringsRecordFormat();
    FormatSpecification formatSpec = new FormatSpecification(DelimitedStringsRecordFormat.class.getCanonicalName(), schema, Collections.<String, String>emptyMap());
    format.initialize(formatSpec);
}

Example 20 with FormatSpecification

use of io.cdap.cdap.api.data.format.FormatSpecification in project cdap by caskdata.

the class DelimitedStringsRecordFormatTest method testFormatRecordWithMapping.

@Test
public void testFormatRecordWithMapping() throws UnsupportedTypeException {
    Schema schema = Schema.recordOf("event", Schema.Field.of("f3", Schema.unionOf(Schema.of(Schema.Type.FLOAT), Schema.of(Schema.Type.NULL))), Schema.Field.of("f4", Schema.unionOf(Schema.of(Schema.Type.DOUBLE), Schema.of(Schema.Type.NULL))));
    DelimitedStringsRecordFormat format = new DelimitedStringsRecordFormat();
    FormatSpecification spec = new FormatSpecification(DelimitedStringsRecordFormat.class.getCanonicalName(), schema, ImmutableMap.of(DelimitedStringsRecordFormat.DELIMITER, ",", DelimitedStringsRecordFormat.MAPPING, "2:f3,3:f4"));
    format.initialize(spec);
    boolean booleanVal = false;
    int intVal = Integer.MAX_VALUE;
    float floatVal = Float.MAX_VALUE;
    double doubleVal = Double.MAX_VALUE;
    byte[] bytesVal = new byte[] { 0, 1, 2 };
    String stringVal = "foo bar";
    String[] arrayVal = new String[] { "extra1", "extra2", "extra3" };
    String body = new StringBuilder().append(booleanVal).append(",").append(intVal).append(",").append(floatVal).append(",").append(doubleVal).append(",").append(Bytes.toStringBinary(bytesVal)).append(",").append(stringVal).append(",").append(arrayVal[0]).append(",").append(arrayVal[1]).append(",").append(arrayVal[2]).toString();
    StructuredRecord output = format.read(ByteBuffer.wrap(Bytes.toBytes(body)));
    Assert.assertEquals(2, output.getSchema().getFields().size());
    Assert.assertNull(output.get("f1"));
    Assert.assertNull(output.get("f2"));
    Assert.assertEquals(floatVal, output.get("f3"), 0.0001f);
    Assert.assertEquals(doubleVal, output.get("f4"), 0.0001d);
    Assert.assertNull(output.get("f5"));
    Assert.assertNull(output.get("f6"));
    Assert.assertNull(output.get("f7"));
    // now try with null fields.
    output = format.read(ByteBuffer.wrap(Bytes.toBytes("true,,3.14159,,,hello world,extra1")));
    Assert.assertEquals(2, output.getSchema().getFields().size());
    Assert.assertNull(output.get("f1"));
    Assert.assertNull(output.get("f2"));
    Assert.assertEquals(3.14159f, output.get("f3"), 0.0001f);
    Assert.assertNull(output.get("f4"));
    Assert.assertNull(output.get("f5"));
    Assert.assertNull(output.get("f6"));
    Assert.assertNull(output.get("f7"));
}