Examples with RecordSchema org.apache.nifi.serialization.record.RecordSchema used on opensource projects

Example 71 with RecordSchema

use of org.apache.nifi.serialization.record.RecordSchema in project nifi by apache.

the class TestJsonTreeRowRecordReader method getAccountSchema.

private RecordSchema getAccountSchema() {
    final List<RecordField> accountFields = new ArrayList<>();
    accountFields.add(new RecordField("id", RecordFieldType.INT.getDataType()));
    accountFields.add(new RecordField("balance", RecordFieldType.DOUBLE.getDataType()));
    final RecordSchema accountSchema = new SimpleRecordSchema(accountFields);
    return accountSchema;
}

Example 72 with RecordSchema

use of org.apache.nifi.serialization.record.RecordSchema in project nifi by apache.

the class TestJsonTreeRowRecordReader method testElementWithNestedData.

@Test
public void testElementWithNestedData() throws IOException, MalformedRecordException {
    final DataType accountType = RecordFieldType.RECORD.getRecordDataType(getAccountSchema());
    final List<RecordField> fields = getDefaultFields();
    fields.add(new RecordField("account", accountType));
    fields.remove(new RecordField("balance", RecordFieldType.DOUBLE.getDataType()));
    final RecordSchema schema = new SimpleRecordSchema(fields);
    try (final InputStream in = new FileInputStream(new File("src/test/resources/json/single-element-nested.json"));
        final JsonTreeRowRecordReader reader = new JsonTreeRowRecordReader(in, Mockito.mock(ComponentLog.class), schema, dateFormat, timeFormat, timestampFormat)) {
        final List<RecordFieldType> dataTypes = schema.getDataTypes().stream().map(dt -> dt.getFieldType()).collect(Collectors.toList());
        final List<RecordFieldType> expectedTypes = Arrays.asList(new RecordFieldType[] { RecordFieldType.INT, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.RECORD });
        assertEquals(expectedTypes, dataTypes);
        final Object[] firstRecordValues = reader.nextRecord().getValues();
        final Object[] allButLast = Arrays.copyOfRange(firstRecordValues, 0, firstRecordValues.length - 1);
        Assert.assertArrayEquals(new Object[] { 1, "John Doe", "123 My Street", "My City", "MS", "11111", "USA" }, allButLast);
        final Object last = firstRecordValues[firstRecordValues.length - 1];
        assertTrue(Record.class.isAssignableFrom(last.getClass()));
        final Record record = (Record) last;
        assertEquals(42, record.getValue("id"));
        assertEquals(4750.89, record.getValue("balance"));
        assertNull(reader.nextRecord());
    }
}

Example 73 with RecordSchema

use of org.apache.nifi.serialization.record.RecordSchema in project nifi by apache.

the class TestJsonTreeRowRecordReader method testReadRawRecordTypeCoercion.

@Test
public void testReadRawRecordTypeCoercion() throws IOException, MalformedRecordException {
    final List<RecordField> fields = new ArrayList<>();
    fields.add(new RecordField("id", RecordFieldType.STRING.getDataType()));
    fields.add(new RecordField("name", RecordFieldType.STRING.getDataType()));
    final RecordSchema schema = new SimpleRecordSchema(fields);
    try (final InputStream in = new FileInputStream(new File("src/test/resources/json/bank-account-array.json"));
        final JsonTreeRowRecordReader reader = new JsonTreeRowRecordReader(in, Mockito.mock(ComponentLog.class), schema, dateFormat, timeFormat, timestampFormat)) {
        final Record schemaValidatedRecord = reader.nextRecord();
        // will be coerced into a STRING as per the schema
        assertEquals("1", schemaValidatedRecord.getValue("id"));
        assertEquals("John Doe", schemaValidatedRecord.getValue("name"));
        assertNull(schemaValidatedRecord.getValue("balance"));
        assertEquals(2, schemaValidatedRecord.getRawFieldNames().size());
    }
    try (final InputStream in = new FileInputStream(new File("src/test/resources/json/bank-account-array.json"));
        final JsonTreeRowRecordReader reader = new JsonTreeRowRecordReader(in, Mockito.mock(ComponentLog.class), schema, dateFormat, timeFormat, timestampFormat)) {
        final Record rawRecord = reader.nextRecord(false, false);
        // will return raw value of (int) 1
        assertEquals(1, rawRecord.getValue("id"));
        assertEquals("John Doe", rawRecord.getValue("name"));
        assertEquals(4750.89, rawRecord.getValue("balance"));
        assertEquals("123 My Street", rawRecord.getValue("address"));
        assertEquals("My City", rawRecord.getValue("city"));
        assertEquals("MS", rawRecord.getValue("state"));
        assertEquals("11111", rawRecord.getValue("zipCode"));
        assertEquals("USA", rawRecord.getValue("country"));
        assertEquals(8, rawRecord.getRawFieldNames().size());
    }
}

Example 74 with RecordSchema

use of org.apache.nifi.serialization.record.RecordSchema in project nifi by apache.

the class TestWriteJsonResult method testExtraFieldInWriteRecord.

@Test
public void testExtraFieldInWriteRecord() throws IOException {
    final List<RecordField> fields = new ArrayList<>();
    fields.add(new RecordField("id", RecordFieldType.STRING.getDataType()));
    final RecordSchema schema = new SimpleRecordSchema(fields);
    final Map<String, Object> values = new HashMap<>();
    values.put("id", "1");
    values.put("name", "John");
    final Record record = new MapRecord(schema, values);
    final ByteArrayOutputStream baos = new ByteArrayOutputStream();
    try (final WriteJsonResult writer = new WriteJsonResult(Mockito.mock(ComponentLog.class), schema, new SchemaNameAsAttribute(), baos, false, NullSuppression.NEVER_SUPPRESS, null, null, null)) {
        writer.beginRecordSet();
        writer.writeRecord(record);
        writer.finishRecordSet();
    }
    final byte[] data = baos.toByteArray();
    final String expected = "[{\"id\":\"1\"}]";
    final String output = new String(data, StandardCharsets.UTF_8);
    assertEquals(expected, output);
}

Example 75 with RecordSchema

use of org.apache.nifi.serialization.record.RecordSchema in project nifi by apache.

the class TestWriteJsonResult method testMissingAndExtraFieldInWriteRawRecord.

@Test
public void testMissingAndExtraFieldInWriteRawRecord() throws IOException {
    final List<RecordField> fields = new ArrayList<>();
    fields.add(new RecordField("id", RecordFieldType.STRING.getDataType()));
    fields.add(new RecordField("name", RecordFieldType.STRING.getDataType()));
    final RecordSchema schema = new SimpleRecordSchema(fields);
    final Map<String, Object> values = new LinkedHashMap<>();
    values.put("id", "1");
    values.put("dob", "1/1/1970");
    final Record record = new MapRecord(schema, values);
    final ByteArrayOutputStream baos = new ByteArrayOutputStream();
    try (final WriteJsonResult writer = new WriteJsonResult(Mockito.mock(ComponentLog.class), schema, new SchemaNameAsAttribute(), baos, false, NullSuppression.NEVER_SUPPRESS, null, null, null)) {
        writer.beginRecordSet();
        writer.writeRawRecord(record);
        writer.finishRecordSet();
    }
    final byte[] data = baos.toByteArray();
    final String expected = "[{\"id\":\"1\",\"dob\":\"1/1/1970\"}]";
    final String output = new String(data, StandardCharsets.UTF_8);
    assertEquals(expected, output);
}