Examples with FormatSpecification co.cask.cdap.api.data.format.FormatSpecification used on opensource projects

Example 51 with FormatSpecification

use of co.cask.cdap.api.data.format.FormatSpecification in project cdap by caskdata.

the class DelimitedStringsRecordFormatTest method testRecordFieldInvalid.

@Test(expected = UnsupportedTypeException.class)
public void testRecordFieldInvalid() throws UnsupportedTypeException {
    Schema recordSchema = Schema.recordOf("record", Schema.Field.of("recordField", Schema.of(Schema.Type.STRING)));
    Schema schema = Schema.recordOf("event", Schema.Field.of("f1", recordSchema));
    FormatSpecification formatSpec = new FormatSpecification(DelimitedStringsRecordFormat.class.getCanonicalName(), schema, Collections.<String, String>emptyMap());
    DelimitedStringsRecordFormat format = new DelimitedStringsRecordFormat();
    format.initialize(formatSpec);
}

Example 52 with FormatSpecification

use of co.cask.cdap.api.data.format.FormatSpecification in project cdap by caskdata.

the class DelimitedStringsRecordFormatTest method testTSV.

@Test
public void testTSV() throws Exception {
    FormatSpecification spec = new FormatSpecification(Formats.TSV, null, Collections.<String, String>emptyMap());
    RecordFormat<StreamEvent, StructuredRecord> format = RecordFormats.createInitializedFormat(spec);
    String body = "userX\tactionY\titemZ";
    StructuredRecord output = format.read(new StreamEvent(ByteBuffer.wrap(Bytes.toBytes(body))));
    String[] actual = output.get("body");
    String[] expected = body.split("\t");
    Assert.assertArrayEquals(expected, actual);
}

Example 53 with FormatSpecification

use of co.cask.cdap.api.data.format.FormatSpecification in project cdap by caskdata.

the class GrokRecordFormatTest method testDefault.

@Test
public void testDefault() throws Exception {
    FormatSpecification spec = new FormatSpecification(Formats.GROK, null, GrokRecordFormat.settings("%{GREEDYDATA:body}"));
    RecordFormat<StreamEvent, StructuredRecord> format = RecordFormats.createInitializedFormat(spec);
    String message = "Oct 17 08:59:00 suod newsyslog[6215]: logfile turned over";
    StructuredRecord record = format.read(new StreamEvent(ByteBuffer.wrap(Bytes.toBytes(message))));
    Assert.assertEquals("Oct 17 08:59:00 suod newsyslog[6215]: logfile turned over", record.get("body"));
}

Example 54 with FormatSpecification

use of co.cask.cdap.api.data.format.FormatSpecification in project cdap by caskdata.

the class DelimitedStringsRecordFormatTest method testNullableFieldsAllowedInSchema.

@Test
public void testNullableFieldsAllowedInSchema() throws UnsupportedTypeException {
    Schema schema = Schema.recordOf("event", Schema.Field.of("f1", Schema.unionOf(Schema.of(Schema.Type.BOOLEAN), Schema.of(Schema.Type.NULL))), Schema.Field.of("f2", Schema.unionOf(Schema.of(Schema.Type.INT), Schema.of(Schema.Type.NULL))), Schema.Field.of("f3", Schema.unionOf(Schema.of(Schema.Type.FLOAT), Schema.of(Schema.Type.NULL))), Schema.Field.of("f4", Schema.unionOf(Schema.of(Schema.Type.DOUBLE), Schema.of(Schema.Type.NULL))), Schema.Field.of("f5", Schema.unionOf(Schema.of(Schema.Type.BYTES), Schema.of(Schema.Type.NULL))), Schema.Field.of("f6", Schema.unionOf(Schema.of(Schema.Type.STRING), Schema.of(Schema.Type.NULL))));
    DelimitedStringsRecordFormat format = new DelimitedStringsRecordFormat();
    FormatSpecification formatSpec = new FormatSpecification(DelimitedStringsRecordFormat.class.getCanonicalName(), schema, Collections.<String, String>emptyMap());
    format.initialize(formatSpec);
}

Example 55 with FormatSpecification

use of co.cask.cdap.api.data.format.FormatSpecification in project cdap by caskdata.

the class DelimitedStringsRecordFormatTest method testRecordMappingWrongMapping.

@Test(expected = IllegalArgumentException.class)
public void testRecordMappingWrongMapping() throws UnsupportedTypeException {
    Schema arraySchema = Schema.arrayOf(Schema.of(Schema.Type.STRING));
    Schema schema = Schema.recordOf("event", Schema.Field.of("f1", arraySchema));
    FormatSpecification formatSpec = new FormatSpecification(DelimitedStringsRecordFormat.class.getCanonicalName(), schema, ImmutableMap.of(DelimitedStringsRecordFormat.MAPPING, "0:f2"));
    DelimitedStringsRecordFormat format = new DelimitedStringsRecordFormat();
    format.initialize(formatSpec);
}