Examples with StructuredRecord io.cdap.cdap.api.data.format.StructuredRecord used on opensource projects

Example 71 with StructuredRecord

use of io.cdap.cdap.api.data.format.StructuredRecord in project cdap by caskdata.

the class GrokRecordFormatTest method testSyslog.

@Test
public void testSyslog() throws Exception {
    FormatSpecification spec = new FormatSpecification(Formats.SYSLOG, null, Collections.<String, String>emptyMap());
    RecordFormat<ByteBuffer, StructuredRecord> format = RecordFormats.createInitializedFormat(spec);
    String message = "Oct 17 08:59:00 suod newsyslog[6215]: logfile turned over";
    StructuredRecord record = format.read(ByteBuffer.wrap(Bytes.toBytes(message)));
    Assert.assertEquals("Oct 17 08:59:00", record.get("timestamp"));
    Assert.assertEquals("suod", record.get("logsource"));
    Assert.assertEquals("newsyslog", record.get("program"));
    Assert.assertEquals("6215", record.get("pid"));
    Assert.assertEquals("logfile turned over", record.get("message"));
    message = "Oct 17 08:59:04 cdr.cs.colorado.edu amd[29648]: " + "noconn option exists, and was turned on! (May cause NFS hangs on some systems...)";
    record = format.read(ByteBuffer.wrap(Bytes.toBytes(message)));
    Assert.assertEquals("Oct 17 08:59:04", record.get("timestamp"));
    Assert.assertEquals("cdr.cs.colorado.edu", record.get("logsource"));
    Assert.assertEquals("amd", record.get("program"));
    Assert.assertEquals("29648", record.get("pid"));
    Assert.assertEquals("noconn option exists, and was turned on! (May cause NFS hangs on some systems...)", record.get("message"));
}

Example 72 with StructuredRecord

use of io.cdap.cdap.api.data.format.StructuredRecord in project cdap by caskdata.

the class GrokRecordFormatTest method testDefault.

@Test
public void testDefault() throws Exception {
    FormatSpecification spec = new FormatSpecification(Formats.GROK, null, GrokRecordFormat.settings("%{GREEDYDATA:body}"));
    RecordFormat<ByteBuffer, StructuredRecord> format = RecordFormats.createInitializedFormat(spec);
    String message = "Oct 17 08:59:00 suod newsyslog[6215]: logfile turned over";
    StructuredRecord record = format.read(ByteBuffer.wrap(Bytes.toBytes(message)));
    Assert.assertEquals("Oct 17 08:59:00 suod newsyslog[6215]: logfile turned over", record.get("body"));
}

Example 73 with StructuredRecord

use of io.cdap.cdap.api.data.format.StructuredRecord in project cdap by caskdata.

the class StructuredRecordBuilderTest method testDateSupport.

@Test
public void testDateSupport() {
    Schema schema = Schema.recordOf("test", Schema.Field.of("id", Schema.of(Schema.Type.INT)), Schema.Field.of("name", Schema.of(Schema.Type.STRING)), Schema.Field.of("date", Schema.of(Schema.LogicalType.DATE)));
    LocalDate expected = LocalDate.of(2002, 11, 18);
    StructuredRecord record = StructuredRecord.builder(schema).set("id", 1).set("name", "test").setDate("date", expected).build();
    LocalDate actual = record.getDate("date");
    Assert.assertEquals(expected, actual);
}

Example 74 with StructuredRecord

use of io.cdap.cdap.api.data.format.StructuredRecord in project cdap by caskdata.

the class StructuredRecordBuilderTest method testNullLogicalType.

@Test
public void testNullLogicalType() {
    Schema schema = Schema.recordOf("test", Schema.Field.of("id", Schema.of(Schema.Type.INT)), Schema.Field.of("name", Schema.of(Schema.Type.STRING)), Schema.Field.of("date", Schema.nullableOf(Schema.of(Schema.LogicalType.DATE))));
    StructuredRecord record = StructuredRecord.builder(schema).set("id", 1).set("name", "test").setDate("date", null).build();
    LocalDate actual = record.getDate("date");
    Assert.assertNull(actual);
}

Example 75 with StructuredRecord

use of io.cdap.cdap.api.data.format.StructuredRecord in project cdap by caskdata.

the class StructuredRecordBuilderTest method testUnexpectedTimestampType.

@Test
public void testUnexpectedTimestampType() {
    Schema schema = Schema.recordOf("test", Schema.Field.of("x", Schema.of(Schema.LogicalType.TIMESTAMP_MILLIS)));
    StructuredRecord record = StructuredRecord.builder(schema).set("x", "2020-01-01 12:00:00").build();
    thrown.expect(ClassCastException.class);
    thrown.expectMessage("Field 'x' is expected to be a timestamp");
    record.getTimestamp("x");
}