Examples with StructuredRecord io.cdap.cdap.api.data.format.StructuredRecord used on opensource projects

Example 66 with StructuredRecord

use of io.cdap.cdap.api.data.format.StructuredRecord in project cdap by caskdata.

the class StructuredRecordStringConverterTest method testPrimitiveArrays.

@SuppressWarnings("AssertEqualsBetweenInconvertibleTypes")
@Test
public void testPrimitiveArrays() throws Exception {
    Schema arraysSchema = Schema.recordOf("arrays", Schema.Field.of("int", Schema.arrayOf(Schema.of(Schema.Type.INT))), Schema.Field.of("long", Schema.arrayOf(Schema.of(Schema.Type.LONG))), Schema.Field.of("float", Schema.arrayOf(Schema.of(Schema.Type.FLOAT))), Schema.Field.of("double", Schema.arrayOf(Schema.of(Schema.Type.DOUBLE))), Schema.Field.of("bool", Schema.arrayOf(Schema.of(Schema.Type.BOOLEAN))));
    StructuredRecord expected = StructuredRecord.builder(arraysSchema).set("int", new int[] { Integer.MIN_VALUE, 0, Integer.MAX_VALUE }).set("long", new long[] { Long.MIN_VALUE, 0L, Long.MAX_VALUE }).set("float", new float[] { Float.MIN_VALUE, 0f, Float.MAX_VALUE }).set("double", new double[] { Double.MIN_VALUE, 0d, Double.MAX_VALUE }).set("bool", new boolean[] { false, true }).build();
    String recordOfJson = StructuredRecordStringConverter.toJsonString(expected);
    StructuredRecord actual = StructuredRecordStringConverter.fromJsonString(recordOfJson, arraysSchema);
    List<Integer> expectedInts = ImmutableList.of(Integer.MIN_VALUE, 0, Integer.MAX_VALUE);
    List<Long> expectedLongs = ImmutableList.of(Long.MIN_VALUE, 0L, Long.MAX_VALUE);
    List<Float> expectedFloats = ImmutableList.of(Float.MIN_VALUE, 0f, Float.MAX_VALUE);
    List<Double> expectedDoubles = ImmutableList.of(Double.MIN_VALUE, 0d, Double.MAX_VALUE);
    List<Boolean> expectedBools = ImmutableList.of(false, true);
    Assert.assertEquals(expectedInts, actual.get("int"));
    Assert.assertEquals(expectedLongs, actual.get("long"));
    Assert.assertEquals(expectedFloats, actual.get("float"));
    Assert.assertEquals(expectedDoubles, actual.get("double"));
    Assert.assertEquals(expectedBools, actual.get("bool"));
}

Example 67 with StructuredRecord

use of io.cdap.cdap.api.data.format.StructuredRecord in project cdap by caskdata.

the class StructuredRecordStringConverterTest method testDateLogicalTypeConversion.

@Test
public void testDateLogicalTypeConversion() throws Exception {
    Schema dateSchema = Schema.recordOf("date", Schema.Field.of("id", Schema.of(Schema.Type.INT)), Schema.Field.of("name", Schema.nullableOf(Schema.of(Schema.Type.STRING))), Schema.Field.of("date", Schema.nullableOf(Schema.of(Schema.LogicalType.DATE))));
    StructuredRecord record = StructuredRecord.builder(dateSchema).set("id", 1).set("name", "alice").setDate("date", LocalDate.now()).build();
    String jsonOfRecord = StructuredRecordStringConverter.toJsonString(record);
    StructuredRecord recordOfJson = StructuredRecordStringConverter.fromJsonString(jsonOfRecord, dateSchema);
    assertRecordsEqual(record, recordOfJson);
}

Example 68 with StructuredRecord

use of io.cdap.cdap.api.data.format.StructuredRecord in project cdap by caskdata.

the class StructuredRecordStringConverterTest method testInvalidDatetimeLogicalTypeConversion.

// Ignore till CDAP-15317 is fixed
@Ignore
@Test
public void testInvalidDatetimeLogicalTypeConversion() throws Exception {
    Schema dateSchema = Schema.recordOf("date", Schema.Field.of("id", Schema.of(Schema.Type.INT)), Schema.Field.of("name", Schema.nullableOf(Schema.of(Schema.Type.STRING))), Schema.Field.of("datetime_field", Schema.nullableOf(Schema.of(Schema.LogicalType.DATETIME))));
    StructuredRecord record = StructuredRecord.builder(dateSchema).set("id", 1).set("name", "alice").set("datetime_field", "random").build();
    thrown.expect(IOException.class);
    thrown.expectMessage("Unsupported format. DateTime type should be in ISO-8601 format.");
    StructuredRecordStringConverter.toJsonString(record);
}

Example 69 with StructuredRecord

use of io.cdap.cdap.api.data.format.StructuredRecord in project cdap by caskdata.

the class AvroRecordFormatTest method testFlatRecord.

@Test
public void testFlatRecord() throws Exception {
    Schema schema = Schema.recordOf("record", Schema.Field.of("int", Schema.of(Schema.Type.INT)), Schema.Field.of("long", Schema.of(Schema.Type.LONG)), Schema.Field.of("boolean", Schema.of(Schema.Type.BOOLEAN)), Schema.Field.of("bytes", Schema.of(Schema.Type.BYTES)), Schema.Field.of("double", Schema.of(Schema.Type.DOUBLE)), Schema.Field.of("float", Schema.of(Schema.Type.FLOAT)), Schema.Field.of("string", Schema.of(Schema.Type.STRING)), Schema.Field.of("array", Schema.arrayOf(Schema.of(Schema.Type.INT))), Schema.Field.of("map", Schema.mapOf(Schema.of(Schema.Type.STRING), Schema.of(Schema.Type.INT))), Schema.Field.of("nullable", Schema.unionOf(Schema.of(Schema.Type.STRING), Schema.of(Schema.Type.NULL))), Schema.Field.of("nullable2", Schema.unionOf(Schema.of(Schema.Type.STRING), Schema.of(Schema.Type.NULL))));
    FormatSpecification formatSpecification = new FormatSpecification(Formats.AVRO, schema, Collections.emptyMap());
    org.apache.avro.Schema avroSchema = convertSchema(schema);
    GenericRecord record = new GenericRecordBuilder(avroSchema).set("int", Integer.MAX_VALUE).set("long", Long.MAX_VALUE).set("boolean", false).set("bytes", Charsets.UTF_8.encode("hello world")).set("double", Double.MAX_VALUE).set("float", Float.MAX_VALUE).set("string", "foo bar").set("array", Lists.newArrayList(1, 2, 3)).set("map", ImmutableMap.of("k1", 1, "k2", 2)).set("nullable", null).set("nullable2", "Hello").build();
    RecordFormat<ByteBuffer, StructuredRecord> format = RecordFormats.createInitializedFormat(formatSpecification);
    StructuredRecord actual = format.read(toByteBuffer(record));
    Assert.assertEquals(Integer.MAX_VALUE, (int) actual.get("int"));
    Assert.assertEquals(Long.MAX_VALUE, (long) actual.get("long"));
    Assert.assertFalse(actual.get("boolean"));
    Assert.assertArrayEquals(Bytes.toBytes("hello world"), Bytes.toBytes((ByteBuffer) actual.get("bytes")));
    Assert.assertEquals(Double.MAX_VALUE, actual.get("double"), 0.0001d);
    Assert.assertEquals(Float.MAX_VALUE, actual.get("float"), 0.0001f);
    Assert.assertEquals("foo bar", actual.get("string"));
    Assert.assertEquals(Lists.newArrayList(1, 2, 3), actual.get("array"));
    assertMapEquals(ImmutableMap.of("k1", 1, "k2", 2), actual.get("map"));
    Assert.assertNull(actual.get("nullable"));
    Assert.assertEquals("Hello", actual.get("nullable2"));
}

Example 70 with StructuredRecord

use of io.cdap.cdap.api.data.format.StructuredRecord in project cdap by caskdata.

the class DelimitedStringsRecordFormatTest method testDelimiter.

@Test
public void testDelimiter() throws UnsupportedTypeException, UnexpectedFormatException {
    DelimitedStringsRecordFormat format = new DelimitedStringsRecordFormat();
    FormatSpecification spec = new FormatSpecification(DelimitedStringsRecordFormat.class.getCanonicalName(), null, ImmutableMap.of(DelimitedStringsRecordFormat.DELIMITER, " "));
    format.initialize(spec);
    String body = "userX actionY itemZ";
    StructuredRecord output = format.read(ByteBuffer.wrap(Bytes.toBytes(body)));
    String[] actual = output.get("body");
    String[] expected = body.split(" ");
    Assert.assertArrayEquals(expected, actual);
}