Examples with RecordSchema org.apache.nifi.serialization.record.RecordSchema used on opensource projects

Example 66 with RecordSchema

use of org.apache.nifi.serialization.record.RecordSchema in project nifi by apache.

the class TestJsonTreeRowRecordReader method testReadArrayDifferentSchemasWithOptionalElementOverridden.

@Test
public void testReadArrayDifferentSchemasWithOptionalElementOverridden() throws IOException, MalformedRecordException {
    final RecordSchema schema = new SimpleRecordSchema(getDefaultFields());
    try (final InputStream in = new FileInputStream(new File("src/test/resources/json/bank-account-array-optional-balance.json"));
        final JsonTreeRowRecordReader reader = new JsonTreeRowRecordReader(in, Mockito.mock(ComponentLog.class), schema, dateFormat, timeFormat, timestampFormat)) {
        final List<String> fieldNames = schema.getFieldNames();
        final List<String> expectedFieldNames = Arrays.asList(new String[] { "id", "name", "balance", "address", "city", "state", "zipCode", "country" });
        assertEquals(expectedFieldNames, fieldNames);
        final List<RecordFieldType> dataTypes = schema.getDataTypes().stream().map(dt -> dt.getFieldType()).collect(Collectors.toList());
        final List<RecordFieldType> expectedTypes = Arrays.asList(new RecordFieldType[] { RecordFieldType.INT, RecordFieldType.STRING, RecordFieldType.DOUBLE, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING });
        assertEquals(expectedTypes, dataTypes);
        final Object[] firstRecordValues = reader.nextRecord().getValues();
        Assert.assertArrayEquals(new Object[] { 1, "John Doe", 4750.89, "123 My Street", "My City", "MS", "11111", "USA" }, firstRecordValues);
        final Object[] secondRecordValues = reader.nextRecord().getValues();
        Assert.assertArrayEquals(new Object[] { 2, "Jane Doe", null, "321 Your Street", "Your City", "NY", "33333", "USA" }, secondRecordValues);
        final Object[] thirdRecordValues = reader.nextRecord().getValues();
        Assert.assertArrayEquals(new Object[] { 3, "Jimmy Doe", null, "321 Your Street", "Your City", "NY", "33333", "USA" }, thirdRecordValues);
        assertNull(reader.nextRecord());
    }
}

Example 67 with RecordSchema

use of org.apache.nifi.serialization.record.RecordSchema in project nifi by apache.

the class TestJsonTreeRowRecordReader method testReadArray.

@Test
public void testReadArray() throws IOException, MalformedRecordException {
    final RecordSchema schema = new SimpleRecordSchema(getDefaultFields());
    try (final InputStream in = new FileInputStream(new File("src/test/resources/json/bank-account-array.json"));
        final JsonTreeRowRecordReader reader = new JsonTreeRowRecordReader(in, Mockito.mock(ComponentLog.class), schema, dateFormat, timeFormat, timestampFormat)) {
        final List<String> fieldNames = schema.getFieldNames();
        final List<String> expectedFieldNames = Arrays.asList(new String[] { "id", "name", "balance", "address", "city", "state", "zipCode", "country" });
        assertEquals(expectedFieldNames, fieldNames);
        final List<RecordFieldType> dataTypes = schema.getDataTypes().stream().map(dt -> dt.getFieldType()).collect(Collectors.toList());
        final List<RecordFieldType> expectedTypes = Arrays.asList(new RecordFieldType[] { RecordFieldType.INT, RecordFieldType.STRING, RecordFieldType.DOUBLE, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING });
        assertEquals(expectedTypes, dataTypes);
        final Object[] firstRecordValues = reader.nextRecord().getValues();
        Assert.assertArrayEquals(new Object[] { 1, "John Doe", 4750.89, "123 My Street", "My City", "MS", "11111", "USA" }, firstRecordValues);
        final Object[] secondRecordValues = reader.nextRecord().getValues();
        Assert.assertArrayEquals(new Object[] { 2, "Jane Doe", 4820.09, "321 Your Street", "Your City", "NY", "33333", "USA" }, secondRecordValues);
        assertNull(reader.nextRecord());
    }
}

Example 68 with RecordSchema

use of org.apache.nifi.serialization.record.RecordSchema in project nifi by apache.

the class TestJsonTreeRowRecordReader method testElementWithNestedArray.

@Test
public void testElementWithNestedArray() throws IOException, MalformedRecordException {
    final DataType accountRecordType = RecordFieldType.RECORD.getRecordDataType(getAccountSchema());
    final DataType accountsType = RecordFieldType.ARRAY.getArrayDataType(accountRecordType);
    final List<RecordField> fields = getDefaultFields();
    fields.add(new RecordField("accounts", accountsType));
    fields.remove(new RecordField("balance", RecordFieldType.DOUBLE.getDataType()));
    final RecordSchema schema = new SimpleRecordSchema(fields);
    try (final InputStream in = new FileInputStream(new File("src/test/resources/json/single-element-nested-array.json"));
        final JsonTreeRowRecordReader reader = new JsonTreeRowRecordReader(in, Mockito.mock(ComponentLog.class), schema, dateFormat, timeFormat, timestampFormat)) {
        final List<String> fieldNames = schema.getFieldNames();
        final List<String> expectedFieldNames = Arrays.asList(new String[] { "id", "name", "address", "city", "state", "zipCode", "country", "accounts" });
        assertEquals(expectedFieldNames, fieldNames);
        final List<RecordFieldType> dataTypes = schema.getDataTypes().stream().map(dt -> dt.getFieldType()).collect(Collectors.toList());
        final List<RecordFieldType> expectedTypes = Arrays.asList(new RecordFieldType[] { RecordFieldType.INT, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.ARRAY });
        assertEquals(expectedTypes, dataTypes);
        final Object[] firstRecordValues = reader.nextRecord().getValues();
        final Object[] nonArrayValues = Arrays.copyOfRange(firstRecordValues, 0, firstRecordValues.length - 1);
        Assert.assertArrayEquals(new Object[] { 1, "John Doe", "123 My Street", "My City", "MS", "11111", "USA" }, nonArrayValues);
        final Object lastRecord = firstRecordValues[firstRecordValues.length - 1];
        assertTrue(Object[].class.isAssignableFrom(lastRecord.getClass()));
        assertNull(reader.nextRecord());
    }
}

Example 69 with RecordSchema

use of org.apache.nifi.serialization.record.RecordSchema in project nifi by apache.

the class TestJsonTreeRowRecordReader method testReadArrayDifferentSchemas.

@Test
public void testReadArrayDifferentSchemas() throws IOException, MalformedRecordException {
    final RecordSchema schema = new SimpleRecordSchema(getDefaultFields());
    try (final InputStream in = new FileInputStream(new File("src/test/resources/json/bank-account-array-different-schemas.json"));
        final JsonTreeRowRecordReader reader = new JsonTreeRowRecordReader(in, Mockito.mock(ComponentLog.class), schema, dateFormat, timeFormat, timestampFormat)) {
        final List<String> fieldNames = schema.getFieldNames();
        final List<String> expectedFieldNames = Arrays.asList(new String[] { "id", "name", "balance", "address", "city", "state", "zipCode", "country" });
        assertEquals(expectedFieldNames, fieldNames);
        final List<RecordFieldType> dataTypes = schema.getDataTypes().stream().map(dt -> dt.getFieldType()).collect(Collectors.toList());
        final List<RecordFieldType> expectedTypes = Arrays.asList(new RecordFieldType[] { RecordFieldType.INT, RecordFieldType.STRING, RecordFieldType.DOUBLE, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING });
        assertEquals(expectedTypes, dataTypes);
        final Object[] firstRecordValues = reader.nextRecord().getValues();
        Assert.assertArrayEquals(new Object[] { 1, "John Doe", 4750.89, "123 My Street", "My City", "MS", "11111", "USA" }, firstRecordValues);
        final Object[] secondRecordValues = reader.nextRecord().getValues();
        Assert.assertArrayEquals(new Object[] { 2, "Jane Doe", 4820.09, "321 Your Street", "Your City", "NY", "33333", null }, secondRecordValues);
        final Object[] thirdRecordValues = reader.nextRecord().getValues();
        Assert.assertArrayEquals(new Object[] { 3, "Jake Doe", 4751.89, "124 My Street", "My City", "MS", "11111", "USA" }, thirdRecordValues);
        assertNull(reader.nextRecord());
    }
}

Example 70 with RecordSchema

use of org.apache.nifi.serialization.record.RecordSchema in project nifi by apache.

the class TestJsonTreeRowRecordReader method testSingleJsonElementWithChoiceFields.

@Test
public void testSingleJsonElementWithChoiceFields() throws IOException, MalformedRecordException {
    // Wraps default fields by Choice data type to test mapping to a Choice type.
    final List<RecordField> choiceFields = getDefaultFields().stream().map(f -> new RecordField(f.getFieldName(), RecordFieldType.CHOICE.getChoiceDataType(f.getDataType()))).collect(Collectors.toList());
    final RecordSchema schema = new SimpleRecordSchema(choiceFields);
    try (final InputStream in = new FileInputStream(new File("src/test/resources/json/single-bank-account.json"));
        final JsonTreeRowRecordReader reader = new JsonTreeRowRecordReader(in, Mockito.mock(ComponentLog.class), schema, dateFormat, timeFormat, timestampFormat)) {
        final List<String> fieldNames = schema.getFieldNames();
        final List<String> expectedFieldNames = Arrays.asList(new String[] { "id", "name", "balance", "address", "city", "state", "zipCode", "country" });
        assertEquals(expectedFieldNames, fieldNames);
        final List<RecordFieldType> expectedTypes = Arrays.asList(new RecordFieldType[] { RecordFieldType.INT, RecordFieldType.STRING, RecordFieldType.DOUBLE, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING });
        final List<RecordField> fields = schema.getFields();
        for (int i = 0; i < schema.getFields().size(); i++) {
            assertTrue(fields.get(i).getDataType() instanceof ChoiceDataType);
            final ChoiceDataType choiceDataType = (ChoiceDataType) fields.get(i).getDataType();
            assertEquals(expectedTypes.get(i), choiceDataType.getPossibleSubTypes().get(0).getFieldType());
        }
        final Object[] firstRecordValues = reader.nextRecord().getValues();
        Assert.assertArrayEquals(new Object[] { 1, "John Doe", 4750.89, "123 My Street", "My City", "MS", "11111", "USA" }, firstRecordValues);
        assertNull(reader.nextRecord());
    }
}