Examples with RecordSchema org.apache.nifi.serialization.record.RecordSchema used on opensource projects

Example 61 with RecordSchema

use of org.apache.nifi.serialization.record.RecordSchema in project nifi by apache.

the class TestJsonPathRowRecordReader method testReadArrayDifferentSchemas.

@Test
public void testReadArrayDifferentSchemas() throws IOException, MalformedRecordException {
    final RecordSchema schema = new SimpleRecordSchema(getDefaultFields());
    try (final InputStream in = new FileInputStream(new File("src/test/resources/json/bank-account-array-different-schemas.json"));
        final JsonPathRowRecordReader reader = new JsonPathRowRecordReader(allJsonPaths, schema, in, Mockito.mock(ComponentLog.class), dateFormat, timeFormat, timestampFormat)) {
        final List<String> fieldNames = schema.getFieldNames();
        final List<String> expectedFieldNames = Arrays.asList(new String[] { "id", "name", "balance", "address", "city", "state", "zipCode", "country" });
        assertEquals(expectedFieldNames, fieldNames);
        final List<RecordFieldType> dataTypes = schema.getDataTypes().stream().map(dt -> dt.getFieldType()).collect(Collectors.toList());
        final List<RecordFieldType> expectedTypes = Arrays.asList(new RecordFieldType[] { RecordFieldType.INT, RecordFieldType.STRING, RecordFieldType.DOUBLE, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING });
        assertEquals(expectedTypes, dataTypes);
        final Object[] firstRecordValues = reader.nextRecord().getValues();
        Assert.assertArrayEquals(new Object[] { 1, "John Doe", 4750.89, "123 My Street", "My City", "MS", "11111", "USA" }, firstRecordValues);
        final Object[] secondRecordValues = reader.nextRecord().getValues();
        Assert.assertArrayEquals(new Object[] { 2, "Jane Doe", 4820.09, "321 Your Street", "Your City", "NY", "33333", null }, secondRecordValues);
        final Object[] thirdRecordValues = reader.nextRecord().getValues();
        Assert.assertArrayEquals(new Object[] { 3, "Jake Doe", 4751.89, "124 My Street", "My City", "MS", "11111", "USA" }, thirdRecordValues);
        assertNull(reader.nextRecord());
    }
}

Example 62 with RecordSchema

use of org.apache.nifi.serialization.record.RecordSchema in project nifi by apache.

the class TestJsonPathRowRecordReader method testReadArray.

@Test
public void testReadArray() throws IOException, MalformedRecordException {
    final RecordSchema schema = new SimpleRecordSchema(getDefaultFields());
    try (final InputStream in = new FileInputStream(new File("src/test/resources/json/bank-account-array.json"));
        final JsonPathRowRecordReader reader = new JsonPathRowRecordReader(allJsonPaths, schema, in, Mockito.mock(ComponentLog.class), dateFormat, timeFormat, timestampFormat)) {
        final List<String> fieldNames = schema.getFieldNames();
        final List<String> expectedFieldNames = Arrays.asList(new String[] { "id", "name", "balance", "address", "city", "state", "zipCode", "country" });
        assertEquals(expectedFieldNames, fieldNames);
        final List<RecordFieldType> dataTypes = schema.getDataTypes().stream().map(dt -> dt.getFieldType()).collect(Collectors.toList());
        final List<RecordFieldType> expectedTypes = Arrays.asList(new RecordFieldType[] { RecordFieldType.INT, RecordFieldType.STRING, RecordFieldType.DOUBLE, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING });
        assertEquals(expectedTypes, dataTypes);
        final Object[] firstRecordValues = reader.nextRecord().getValues();
        Assert.assertArrayEquals(new Object[] { 1, "John Doe", 4750.89, "123 My Street", "My City", "MS", "11111", "USA" }, firstRecordValues);
        final Object[] secondRecordValues = reader.nextRecord().getValues();
        Assert.assertArrayEquals(new Object[] { 2, "Jane Doe", 4820.09, "321 Your Street", "Your City", "NY", "33333", "USA" }, secondRecordValues);
        assertNull(reader.nextRecord());
    }
}

Example 63 with RecordSchema

use of org.apache.nifi.serialization.record.RecordSchema in project nifi by apache.

the class TestJsonPathRowRecordReader method testPrimitiveTypeArrays.

@Test
public void testPrimitiveTypeArrays() throws IOException, MalformedRecordException {
    final LinkedHashMap<String, JsonPath> jsonPaths = new LinkedHashMap<>(allJsonPaths);
    jsonPaths.put("accountIds", JsonPath.compile("$.accountIds"));
    final List<RecordField> fields = getDefaultFields();
    final DataType idsType = RecordFieldType.ARRAY.getArrayDataType(RecordFieldType.INT.getDataType());
    fields.add(new RecordField("accountIds", idsType));
    final RecordSchema schema = new SimpleRecordSchema(fields);
    try (final InputStream in = new FileInputStream(new File("src/test/resources/json/primitive-type-array.json"));
        final JsonPathRowRecordReader reader = new JsonPathRowRecordReader(jsonPaths, schema, in, Mockito.mock(ComponentLog.class), dateFormat, timeFormat, timestampFormat)) {
        final List<String> fieldNames = schema.getFieldNames();
        final List<String> expectedFieldNames = Arrays.asList(new String[] { "id", "name", "balance", "address", "city", "state", "zipCode", "country", "accountIds" });
        assertEquals(expectedFieldNames, fieldNames);
        final List<RecordFieldType> dataTypes = schema.getDataTypes().stream().map(dt -> dt.getFieldType()).collect(Collectors.toList());
        final List<RecordFieldType> expectedTypes = Arrays.asList(new RecordFieldType[] { RecordFieldType.INT, RecordFieldType.STRING, RecordFieldType.DOUBLE, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.ARRAY });
        assertEquals(expectedTypes, dataTypes);
        final Object[] firstRecordValues = reader.nextRecord().getValues();
        final Object[] nonArrayValues = Arrays.copyOfRange(firstRecordValues, 0, firstRecordValues.length - 1);
        Assert.assertArrayEquals(new Object[] { 1, "John Doe", 4750.89D, "123 My Street", "My City", "MS", "11111", "USA" }, nonArrayValues);
        final Object lastRecord = firstRecordValues[firstRecordValues.length - 1];
        assertNotNull(lastRecord);
        assertTrue(Object[].class.isAssignableFrom(lastRecord.getClass()));
        final Object[] array = (Object[]) lastRecord;
        Assert.assertArrayEquals(new Object[] { 1, 2, 3 }, array);
        assertNull(reader.nextRecord());
        assertNull(reader.nextRecord());
    }
}

Example 64 with RecordSchema

use of org.apache.nifi.serialization.record.RecordSchema in project nifi by apache.

the class TestJsonTreeRowRecordReader method testReadUnicodeCharacters.

@Test
public void testReadUnicodeCharacters() throws IOException, MalformedRecordException {
    final List<RecordField> fromFields = new ArrayList<>();
    fromFields.add(new RecordField("id", RecordFieldType.LONG.getDataType()));
    fromFields.add(new RecordField("name", RecordFieldType.STRING.getDataType()));
    final RecordSchema fromSchema = new SimpleRecordSchema(fromFields);
    final DataType fromType = RecordFieldType.RECORD.getRecordDataType(fromSchema);
    final List<RecordField> fields = new ArrayList<>();
    fields.add(new RecordField("created_at", RecordFieldType.STRING.getDataType()));
    fields.add(new RecordField("id", RecordFieldType.LONG.getDataType()));
    fields.add(new RecordField("unicode", RecordFieldType.STRING.getDataType()));
    fields.add(new RecordField("from", fromType));
    final RecordSchema schema = new SimpleRecordSchema(fields);
    try (final InputStream in = new FileInputStream(new File("src/test/resources/json/json-with-unicode.json"));
        final JsonTreeRowRecordReader reader = new JsonTreeRowRecordReader(in, Mockito.mock(ComponentLog.class), schema, dateFormat, timeFormat, timestampFormat)) {
        final Object[] firstRecordValues = reader.nextRecord().getValues();
        final Object secondValue = firstRecordValues[1];
        assertTrue(secondValue instanceof Long);
        assertEquals(832036744985577473L, secondValue);
        final Object unicodeValue = firstRecordValues[2];
        assertEquals("\u3061\u3083\u6ce3\u304d\u305d\u3046", unicodeValue);
        assertNull(reader.nextRecord());
    }
}

Example 65 with RecordSchema

use of org.apache.nifi.serialization.record.RecordSchema in project nifi by apache.

the class TestJsonTreeRowRecordReader method testPerformanceOnIndividualMessages.

@Test
@Ignore("Intended only for manual testing to determine performance before/after modifications")
public void testPerformanceOnIndividualMessages() throws IOException, MalformedRecordException {
    final RecordSchema schema = new SimpleRecordSchema(Collections.emptyList());
    final File file = new File("/devel/nifi/nifi-assembly/target/nifi-1.2.0-SNAPSHOT-bin/nifi-1.2.0-SNAPSHOT/1.prov.json");
    final byte[] data = Files.readAllBytes(file.toPath());
    final ComponentLog logger = Mockito.mock(ComponentLog.class);
    int recordCount = 0;
    final int iterations = 1_000_000;
    for (int j = 0; j < 5; j++) {
        final long start = System.nanoTime();
        for (int i = 0; i < iterations; i++) {
            try (final InputStream in = new ByteArrayInputStream(data);
                final JsonTreeRowRecordReader reader = new JsonTreeRowRecordReader(in, logger, schema, dateFormat, timeFormat, timestampFormat)) {
                while (reader.nextRecord() != null) {
                    recordCount++;
                }
            }
        }
        final long nanos = System.nanoTime() - start;
        final long millis = TimeUnit.NANOSECONDS.toMillis(nanos);
        System.out.println("Took " + millis + " millis to read " + recordCount + " records");
    }
}