Examples with RecordField org.apache.nifi.serialization.record.RecordField used on opensource projects

Example 91 with RecordField

use of org.apache.nifi.serialization.record.RecordField in project nifi by apache.

the class TestAvroTypeUtil method testDefaultArrayValuesInRecordsCase2.

/**
 * The issue consists on having an Avro's schema with a default value in an
 * array. See
 * <a href="https://issues.apache.org/jira/browse/NIFI-4893">NIFI-4893</a>.
 * @throws IOException
 *             schema not found.
 */
@Test
public void testDefaultArrayValuesInRecordsCase2() throws IOException {
    Schema avroSchema = new Schema.Parser().parse(getClass().getResourceAsStream("defaultArrayInRecords2.json"));
    GenericRecordBuilder builder = new GenericRecordBuilder(avroSchema);
    Record field1Record = new GenericRecordBuilder(avroSchema.getField("field1").schema()).build();
    builder.set("field1", field1Record);
    Record r = builder.build();
    @SuppressWarnings("unchecked") GenericData.Array<Integer> values = (GenericData.Array<Integer>) ((GenericRecord) r.get("field1")).get("listOfInt");
    assertArrayEquals(new Object[] { 1, 2, 3 }, values.toArray());
    RecordSchema record = AvroTypeUtil.createSchema(avroSchema);
    RecordField field = record.getField("field1").get();
    assertEquals(RecordFieldType.RECORD, field.getDataType().getFieldType());
    RecordDataType data = (RecordDataType) field.getDataType();
    RecordSchema childSchema = data.getChildSchema();
    RecordField childField = childSchema.getField("listOfInt").get();
    assertEquals(RecordFieldType.ARRAY, childField.getDataType().getFieldType());
    assertTrue(childField.getDefaultValue() instanceof Object[]);
    assertArrayEquals(new Object[] { 1, 2, 3 }, ((Object[]) childField.getDefaultValue()));
}

Example 92 with RecordField

use of org.apache.nifi.serialization.record.RecordField in project nifi by apache.

the class TestAvroTypeUtil method testSimpleRecursiveSchema.

@Test
public // referencing 'next')
void testSimpleRecursiveSchema() {
    Schema recursiveSchema = new Schema.Parser().parse("{\n" + "  \"namespace\": \"org.apache.nifi.testing\",\n" + "  \"name\": \"NodeRecord\",\n" + "  \"type\": \"record\",\n" + "  \"fields\": [\n" + "    {\n" + "      \"name\": \"id\",\n" + "      \"type\": \"int\"\n" + "    },\n" + "    {\n" + "      \"name\": \"value\",\n" + "      \"type\": \"string\"\n" + "    },\n" + "    {\n" + "      \"name\": \"parent\",\n" + "      \"type\": [\n" + "        \"null\",\n" + "        \"NodeRecord\"\n" + "      ]\n" + "    }\n" + "  ]\n" + "}\n");
    // Make sure the following doesn't throw an exception
    RecordSchema result = AvroTypeUtil.createSchema(recursiveSchema);
    // Make sure it parsed correctly
    Assert.assertEquals(3, result.getFieldCount());
    Optional<RecordField> idField = result.getField("id");
    Assert.assertTrue(idField.isPresent());
    Assert.assertEquals(RecordFieldType.INT, idField.get().getDataType().getFieldType());
    Optional<RecordField> valueField = result.getField("value");
    Assert.assertTrue(valueField.isPresent());
    Assert.assertEquals(RecordFieldType.STRING, valueField.get().getDataType().getFieldType());
    Optional<RecordField> parentField = result.getField("parent");
    Assert.assertTrue(parentField.isPresent());
    Assert.assertEquals(RecordFieldType.RECORD, parentField.get().getDataType().getFieldType());
    // The 'parent' field should have a circular schema reference to the top level
    // record schema, similar to how Avro handles this
    Assert.assertEquals(result, ((RecordDataType) parentField.get().getDataType()).getChildSchema());
}

Example 93 with RecordField

use of org.apache.nifi.serialization.record.RecordField in project nifi by apache.

the class TestAvroTypeUtil method testCreateAvroSchemaPrimitiveTypes.

@Test
public void testCreateAvroSchemaPrimitiveTypes() throws SchemaNotFoundException {
    final List<RecordField> fields = new ArrayList<>();
    fields.add(new RecordField("int", RecordFieldType.INT.getDataType()));
    fields.add(new RecordField("long", RecordFieldType.LONG.getDataType()));
    fields.add(new RecordField("string", RecordFieldType.STRING.getDataType(), "hola", Collections.singleton("greeting")));
    fields.add(new RecordField("byte", RecordFieldType.BYTE.getDataType()));
    fields.add(new RecordField("char", RecordFieldType.CHAR.getDataType()));
    fields.add(new RecordField("short", RecordFieldType.SHORT.getDataType()));
    fields.add(new RecordField("double", RecordFieldType.DOUBLE.getDataType()));
    fields.add(new RecordField("float", RecordFieldType.FLOAT.getDataType()));
    fields.add(new RecordField("time", RecordFieldType.TIME.getDataType()));
    fields.add(new RecordField("date", RecordFieldType.DATE.getDataType()));
    fields.add(new RecordField("timestamp", RecordFieldType.TIMESTAMP.getDataType()));
    final DataType arrayType = RecordFieldType.ARRAY.getArrayDataType(RecordFieldType.STRING.getDataType());
    fields.add(new RecordField("strings", arrayType));
    final DataType mapType = RecordFieldType.MAP.getMapDataType(RecordFieldType.LONG.getDataType());
    fields.add(new RecordField("map", mapType));
    final List<RecordField> personFields = new ArrayList<>();
    personFields.add(new RecordField("name", RecordFieldType.STRING.getDataType()));
    personFields.add(new RecordField("dob", RecordFieldType.DATE.getDataType()));
    final RecordSchema personSchema = new SimpleRecordSchema(personFields);
    final DataType personType = RecordFieldType.RECORD.getRecordDataType(personSchema);
    fields.add(new RecordField("person", personType));
    final RecordSchema recordSchema = new SimpleRecordSchema(fields);
    final Schema avroSchema = AvroTypeUtil.extractAvroSchema(recordSchema);
    // everything should be a union, since it's nullable.
    for (final Field field : avroSchema.getFields()) {
        final Schema fieldSchema = field.schema();
        assertEquals(Type.UNION, fieldSchema.getType());
        assertTrue("Field " + field.name() + " does not contain NULL type", fieldSchema.getTypes().contains(Schema.create(Type.NULL)));
    }
    final RecordSchema afterConversion = AvroTypeUtil.createSchema(avroSchema);
    assertEquals(RecordFieldType.INT.getDataType(), afterConversion.getDataType("int").get());
    assertEquals(RecordFieldType.LONG.getDataType(), afterConversion.getDataType("long").get());
    assertEquals(RecordFieldType.STRING.getDataType(), afterConversion.getDataType("string").get());
    assertEquals(RecordFieldType.INT.getDataType(), afterConversion.getDataType("byte").get());
    assertEquals(RecordFieldType.STRING.getDataType(), afterConversion.getDataType("char").get());
    assertEquals(RecordFieldType.INT.getDataType(), afterConversion.getDataType("short").get());
    assertEquals(RecordFieldType.DOUBLE.getDataType(), afterConversion.getDataType("double").get());
    assertEquals(RecordFieldType.FLOAT.getDataType(), afterConversion.getDataType("float").get());
    assertEquals(RecordFieldType.TIME.getDataType(), afterConversion.getDataType("time").get());
    assertEquals(RecordFieldType.DATE.getDataType(), afterConversion.getDataType("date").get());
    assertEquals(RecordFieldType.TIMESTAMP.getDataType(), afterConversion.getDataType("timestamp").get());
    assertEquals(arrayType, afterConversion.getDataType("strings").get());
    assertEquals(mapType, afterConversion.getDataType("map").get());
    assertEquals(personType, afterConversion.getDataType("person").get());
    final RecordField stringField = afterConversion.getField("string").get();
    assertEquals("hola", stringField.getDefaultValue());
    assertEquals(Collections.singleton("greeting"), stringField.getAliases());
}

Example 94 with RecordField

use of org.apache.nifi.serialization.record.RecordField in project nifi by apache.

the class TestAvroTypeUtil method testDefaultArrayValue1.

/**
 * The issue consists on having an Avro's schema with a default value in an
 * array. See
 * <a href="https://issues.apache.org/jira/browse/NIFI-4893">NIFI-4893</a>.
 * @throws IOException
 *             schema not found.
 */
@Test
public void testDefaultArrayValue1() throws IOException {
    Schema avroSchema = new Schema.Parser().parse(getClass().getResourceAsStream("defaultArrayValue1.json"));
    GenericRecordBuilder builder = new GenericRecordBuilder(avroSchema);
    Record r = builder.build();
    @SuppressWarnings("unchecked") GenericData.Array<Integer> values = (GenericData.Array<Integer>) r.get("listOfInt");
    assertEquals(values.size(), 0);
    RecordSchema record = AvroTypeUtil.createSchema(avroSchema);
    RecordField field = record.getField("listOfInt").get();
    assertEquals(RecordFieldType.ARRAY, field.getDataType().getFieldType());
    assertTrue(field.getDefaultValue() instanceof Object[]);
    assertEquals(0, ((Object[]) field.getDefaultValue()).length);
}

Example 95 with RecordField

use of org.apache.nifi.serialization.record.RecordField in project nifi by apache.

the class TestAvroTypeUtil method testComplicatedRecursiveSchema.

@Test
public // A
void testComplicatedRecursiveSchema() {
    Schema recursiveSchema = new Schema.Parser().parse("{\n" + "  \"namespace\": \"org.apache.nifi.testing\",\n" + "  \"name\": \"Record_A\",\n" + "  \"type\": \"record\",\n" + "  \"fields\": [\n" + "    {\n" + "      \"name\": \"id\",\n" + "      \"type\": \"int\"\n" + "    },\n" + "    {\n" + "      \"name\": \"value\",\n" + "      \"type\": \"string\"\n" + "    },\n" + "    {\n" + "      \"name\": \"child\",\n" + "      \"type\": {\n" + "        \"namespace\": \"org.apache.nifi.testing\",\n" + "        \"name\": \"Record_B\",\n" + "        \"type\": \"record\",\n" + "        \"fields\": [\n" + "          {\n" + "            \"name\": \"id\",\n" + "            \"type\": \"int\"\n" + "          },\n" + "          {\n" + "            \"name\": \"value\",\n" + "            \"type\": \"string\"\n" + "          },\n" + "          {\n" + "            \"name\": \"parent\",\n" + "            \"type\": [\n" + "              \"null\",\n" + "              \"Record_A\"\n" + "            ]\n" + "          }\n" + "        ]\n" + "      }\n" + "    }\n" + "  ]\n" + "}\n");
    // Make sure the following doesn't throw an exception
    RecordSchema recordASchema = AvroTypeUtil.createSchema(recursiveSchema);
    // Make sure it parsed correctly
    Assert.assertEquals(3, recordASchema.getFieldCount());
    Optional<RecordField> recordAIdField = recordASchema.getField("id");
    Assert.assertTrue(recordAIdField.isPresent());
    Assert.assertEquals(RecordFieldType.INT, recordAIdField.get().getDataType().getFieldType());
    Optional<RecordField> recordAValueField = recordASchema.getField("value");
    Assert.assertTrue(recordAValueField.isPresent());
    Assert.assertEquals(RecordFieldType.STRING, recordAValueField.get().getDataType().getFieldType());
    Optional<RecordField> recordAChildField = recordASchema.getField("child");
    Assert.assertTrue(recordAChildField.isPresent());
    Assert.assertEquals(RecordFieldType.RECORD, recordAChildField.get().getDataType().getFieldType());
    // Get the child schema
    RecordSchema recordBSchema = ((RecordDataType) recordAChildField.get().getDataType()).getChildSchema();
    // Make sure it parsed correctly
    Assert.assertEquals(3, recordBSchema.getFieldCount());
    Optional<RecordField> recordBIdField = recordBSchema.getField("id");
    Assert.assertTrue(recordBIdField.isPresent());
    Assert.assertEquals(RecordFieldType.INT, recordBIdField.get().getDataType().getFieldType());
    Optional<RecordField> recordBValueField = recordBSchema.getField("value");
    Assert.assertTrue(recordBValueField.isPresent());
    Assert.assertEquals(RecordFieldType.STRING, recordBValueField.get().getDataType().getFieldType());
    Optional<RecordField> recordBParentField = recordBSchema.getField("parent");
    Assert.assertTrue(recordBParentField.isPresent());
    Assert.assertEquals(RecordFieldType.RECORD, recordBParentField.get().getDataType().getFieldType());
    // Make sure the 'parent' field has a schema reference back to the original top
    // level record schema
    Assert.assertEquals(recordASchema, ((RecordDataType) recordBParentField.get().getDataType()).getChildSchema());
}