Examples with FormatSpecification io.cdap.cdap.api.data.format.FormatSpecification used on opensource projects

Example 6 with FormatSpecification

use of io.cdap.cdap.api.data.format.FormatSpecification in project cdap by caskdata.

the class DelimitedStringsRecordFormatTest method testSimpleSchemaValidation.

@Test
public void testSimpleSchemaValidation() throws UnsupportedTypeException {
    Schema simpleSchema = Schema.recordOf("event", Schema.Field.of("f1", Schema.of(Schema.Type.BOOLEAN)), Schema.Field.of("f2", Schema.of(Schema.Type.INT)), Schema.Field.of("f3", Schema.of(Schema.Type.FLOAT)), Schema.Field.of("f4", Schema.of(Schema.Type.DOUBLE)), Schema.Field.of("f5", Schema.of(Schema.Type.BYTES)), Schema.Field.of("f6", Schema.of(Schema.Type.STRING)));
    DelimitedStringsRecordFormat format = new DelimitedStringsRecordFormat();
    FormatSpecification formatSpec = new FormatSpecification(DelimitedStringsRecordFormat.class.getCanonicalName(), simpleSchema, Collections.<String, String>emptyMap());
    format.initialize(formatSpec);
}

Example 7 with FormatSpecification

use of io.cdap.cdap.api.data.format.FormatSpecification in project cdap by caskdata.

the class DelimitedStringsRecordFormatTest method testComplexArraySchemaValidation.

@Test(expected = UnsupportedTypeException.class)
public void testComplexArraySchemaValidation() throws UnsupportedTypeException {
    Schema mapSchema = Schema.mapOf(Schema.of(Schema.Type.STRING), Schema.of(Schema.Type.STRING));
    Schema schema = Schema.recordOf("event", Schema.Field.of("f1", Schema.arrayOf(mapSchema)));
    FormatSpecification formatSpec = new FormatSpecification(DelimitedStringsRecordFormat.class.getCanonicalName(), schema, Collections.<String, String>emptyMap());
    DelimitedStringsRecordFormat format = new DelimitedStringsRecordFormat();
    format.initialize(formatSpec);
}

Example 8 with FormatSpecification

use of io.cdap.cdap.api.data.format.FormatSpecification in project cdap by caskdata.

the class DelimitedStringsRecordFormatTest method testDelimiter.

@Test
public void testDelimiter() throws UnsupportedTypeException, UnexpectedFormatException {
    DelimitedStringsRecordFormat format = new DelimitedStringsRecordFormat();
    FormatSpecification spec = new FormatSpecification(DelimitedStringsRecordFormat.class.getCanonicalName(), null, ImmutableMap.of(DelimitedStringsRecordFormat.DELIMITER, " "));
    format.initialize(spec);
    String body = "userX actionY itemZ";
    StructuredRecord output = format.read(ByteBuffer.wrap(Bytes.toBytes(body)));
    String[] actual = output.get("body");
    String[] expected = body.split(" ");
    Assert.assertArrayEquals(expected, actual);
}

Example 9 with FormatSpecification

use of io.cdap.cdap.api.data.format.FormatSpecification in project cdap by caskdata.

the class GrokRecordFormatTest method testSyslog.

@Test
public void testSyslog() throws Exception {
    FormatSpecification spec = new FormatSpecification(Formats.SYSLOG, null, Collections.<String, String>emptyMap());
    RecordFormat<ByteBuffer, StructuredRecord> format = RecordFormats.createInitializedFormat(spec);
    String message = "Oct 17 08:59:00 suod newsyslog[6215]: logfile turned over";
    StructuredRecord record = format.read(ByteBuffer.wrap(Bytes.toBytes(message)));
    Assert.assertEquals("Oct 17 08:59:00", record.get("timestamp"));
    Assert.assertEquals("suod", record.get("logsource"));
    Assert.assertEquals("newsyslog", record.get("program"));
    Assert.assertEquals("6215", record.get("pid"));
    Assert.assertEquals("logfile turned over", record.get("message"));
    message = "Oct 17 08:59:04 cdr.cs.colorado.edu amd[29648]: " + "noconn option exists, and was turned on! (May cause NFS hangs on some systems...)";
    record = format.read(ByteBuffer.wrap(Bytes.toBytes(message)));
    Assert.assertEquals("Oct 17 08:59:04", record.get("timestamp"));
    Assert.assertEquals("cdr.cs.colorado.edu", record.get("logsource"));
    Assert.assertEquals("amd", record.get("program"));
    Assert.assertEquals("29648", record.get("pid"));
    Assert.assertEquals("noconn option exists, and was turned on! (May cause NFS hangs on some systems...)", record.get("message"));
}

Example 10 with FormatSpecification

use of io.cdap.cdap.api.data.format.FormatSpecification in project cdap by caskdata.

the class GrokRecordFormatTest method testDefault.

@Test
public void testDefault() throws Exception {
    FormatSpecification spec = new FormatSpecification(Formats.GROK, null, GrokRecordFormat.settings("%{GREEDYDATA:body}"));
    RecordFormat<ByteBuffer, StructuredRecord> format = RecordFormats.createInitializedFormat(spec);
    String message = "Oct 17 08:59:00 suod newsyslog[6215]: logfile turned over";
    StructuredRecord record = format.read(ByteBuffer.wrap(Bytes.toBytes(message)));
    Assert.assertEquals("Oct 17 08:59:00 suod newsyslog[6215]: logfile turned over", record.get("body"));
}