Examples with RecordFieldType org.apache.nifi.serialization.record.RecordFieldType used on opensource projects

Example 1 with RecordFieldType

use of org.apache.nifi.serialization.record.RecordFieldType in project nifi by apache.

the class TestWriteCSVResult method testDataTypes.

@Test
public void testDataTypes() throws IOException {
    final CSVFormat csvFormat = CSVFormat.DEFAULT.withQuoteMode(QuoteMode.ALL).withRecordSeparator("\n");
    final StringBuilder headerBuilder = new StringBuilder();
    final List<RecordField> fields = new ArrayList<>();
    for (final RecordFieldType fieldType : RecordFieldType.values()) {
        if (fieldType == RecordFieldType.CHOICE) {
            final List<DataType> possibleTypes = new ArrayList<>();
            possibleTypes.add(RecordFieldType.INT.getDataType());
            possibleTypes.add(RecordFieldType.LONG.getDataType());
            fields.add(new RecordField(fieldType.name().toLowerCase(), fieldType.getChoiceDataType(possibleTypes)));
        } else {
            fields.add(new RecordField(fieldType.name().toLowerCase(), fieldType.getDataType()));
        }
        headerBuilder.append('"').append(fieldType.name().toLowerCase()).append('"').append(",");
    }
    final RecordSchema schema = new SimpleRecordSchema(fields);
    final ByteArrayOutputStream baos = new ByteArrayOutputStream();
    final long now = System.currentTimeMillis();
    try (final WriteCSVResult result = new WriteCSVResult(csvFormat, schema, new SchemaNameAsAttribute(), baos, RecordFieldType.DATE.getDefaultFormat(), RecordFieldType.TIME.getDefaultFormat(), RecordFieldType.TIMESTAMP.getDefaultFormat(), true, "UTF-8")) {
        final Map<String, Object> valueMap = new HashMap<>();
        valueMap.put("string", "a孟bc李12儒3");
        valueMap.put("boolean", true);
        valueMap.put("byte", (byte) 1);
        valueMap.put("char", 'c');
        valueMap.put("short", (short) 8);
        valueMap.put("int", 9);
        valueMap.put("bigint", BigInteger.valueOf(8L));
        valueMap.put("long", 8L);
        valueMap.put("float", 8.0F);
        valueMap.put("double", 8.0D);
        valueMap.put("date", new Date(now));
        valueMap.put("time", new Time(now));
        valueMap.put("timestamp", new Timestamp(now));
        valueMap.put("record", null);
        valueMap.put("choice", 48L);
        valueMap.put("array", null);
        final Record record = new MapRecord(schema, valueMap);
        final RecordSet rs = RecordSet.of(schema, record);
        result.write(rs);
    }
    final String output = new String(baos.toByteArray(), StandardCharsets.UTF_8);
    headerBuilder.deleteCharAt(headerBuilder.length() - 1);
    final String headerLine = headerBuilder.toString();
    final String[] splits = output.split("\n");
    assertEquals(2, splits.length);
    assertEquals(headerLine, splits[0]);
    final String values = splits[1];
    final StringBuilder expectedBuilder = new StringBuilder();
    expectedBuilder.append("\"a孟bc李12儒3\",\"true\",\"1\",\"c\",\"8\",\"9\",\"8\",\"8\",\"8.0\",\"8.0\",");
    final String dateValue = getDateFormat(RecordFieldType.DATE.getDefaultFormat()).format(now);
    final String timeValue = getDateFormat(RecordFieldType.TIME.getDefaultFormat()).format(now);
    final String timestampValue = getDateFormat(RecordFieldType.TIMESTAMP.getDefaultFormat()).format(now);
    expectedBuilder.append('"').append(dateValue).append('"').append(',');
    expectedBuilder.append('"').append(timeValue).append('"').append(',');
    expectedBuilder.append('"').append(timestampValue).append('"').append(',');
    expectedBuilder.append(",\"48\",,");
    final String expectedValues = expectedBuilder.toString();
    assertEquals(expectedValues, values);
}

Example 2 with RecordFieldType

use of org.apache.nifi.serialization.record.RecordFieldType in project nifi by apache.

the class TestJsonPathRowRecordReader method testElementWithNestedArray.

@Test
public void testElementWithNestedArray() throws IOException, MalformedRecordException {
    final LinkedHashMap<String, JsonPath> jsonPaths = new LinkedHashMap<>(allJsonPaths);
    jsonPaths.put("accounts", JsonPath.compile("$.accounts"));
    final DataType accountRecordType = RecordFieldType.RECORD.getRecordDataType(getAccountSchema());
    final DataType accountsType = RecordFieldType.ARRAY.getArrayDataType(accountRecordType);
    final List<RecordField> fields = getDefaultFields();
    fields.add(new RecordField("accounts", accountsType));
    final RecordSchema schema = new SimpleRecordSchema(fields);
    try (final InputStream in = new FileInputStream(new File("src/test/resources/json/single-element-nested-array.json"));
        final JsonPathRowRecordReader reader = new JsonPathRowRecordReader(jsonPaths, schema, in, Mockito.mock(ComponentLog.class), dateFormat, timeFormat, timestampFormat)) {
        final List<String> fieldNames = schema.getFieldNames();
        final List<String> expectedFieldNames = Arrays.asList(new String[] { "id", "name", "balance", "address", "city", "state", "zipCode", "country", "accounts" });
        assertEquals(expectedFieldNames, fieldNames);
        final List<RecordFieldType> dataTypes = schema.getDataTypes().stream().map(dt -> dt.getFieldType()).collect(Collectors.toList());
        final List<RecordFieldType> expectedTypes = Arrays.asList(new RecordFieldType[] { RecordFieldType.INT, RecordFieldType.STRING, RecordFieldType.DOUBLE, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.ARRAY });
        assertEquals(expectedTypes, dataTypes);
        final Object[] firstRecordValues = reader.nextRecord().getValues();
        final Object[] nonArrayValues = Arrays.copyOfRange(firstRecordValues, 0, firstRecordValues.length - 1);
        Assert.assertArrayEquals(new Object[] { 1, "John Doe", null, "123 My Street", "My City", "MS", "11111", "USA" }, nonArrayValues);
        final Object lastRecord = firstRecordValues[firstRecordValues.length - 1];
        assertTrue(Object[].class.isAssignableFrom(lastRecord.getClass()));
        final Object[] array = (Object[]) lastRecord;
        assertEquals(2, array.length);
        final Object firstElement = array[0];
        assertTrue(firstElement instanceof Record);
        final Record firstRecord = (Record) firstElement;
        assertEquals(42, firstRecord.getValue("id"));
        assertEquals(4750.89D, firstRecord.getValue("balance"));
        final Object secondElement = array[1];
        assertTrue(secondElement instanceof Record);
        final Record secondRecord = (Record) secondElement;
        assertEquals(43, secondRecord.getValue("id"));
        assertEquals(48212.38D, secondRecord.getValue("balance"));
        assertNull(reader.nextRecord());
    }
}

Example 3 with RecordFieldType

use of org.apache.nifi.serialization.record.RecordFieldType in project nifi by apache.

the class TestJsonPathRowRecordReader method testReadArrayDifferentSchemas.

@Test
public void testReadArrayDifferentSchemas() throws IOException, MalformedRecordException {
    final RecordSchema schema = new SimpleRecordSchema(getDefaultFields());
    try (final InputStream in = new FileInputStream(new File("src/test/resources/json/bank-account-array-different-schemas.json"));
        final JsonPathRowRecordReader reader = new JsonPathRowRecordReader(allJsonPaths, schema, in, Mockito.mock(ComponentLog.class), dateFormat, timeFormat, timestampFormat)) {
        final List<String> fieldNames = schema.getFieldNames();
        final List<String> expectedFieldNames = Arrays.asList(new String[] { "id", "name", "balance", "address", "city", "state", "zipCode", "country" });
        assertEquals(expectedFieldNames, fieldNames);
        final List<RecordFieldType> dataTypes = schema.getDataTypes().stream().map(dt -> dt.getFieldType()).collect(Collectors.toList());
        final List<RecordFieldType> expectedTypes = Arrays.asList(new RecordFieldType[] { RecordFieldType.INT, RecordFieldType.STRING, RecordFieldType.DOUBLE, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING });
        assertEquals(expectedTypes, dataTypes);
        final Object[] firstRecordValues = reader.nextRecord().getValues();
        Assert.assertArrayEquals(new Object[] { 1, "John Doe", 4750.89, "123 My Street", "My City", "MS", "11111", "USA" }, firstRecordValues);
        final Object[] secondRecordValues = reader.nextRecord().getValues();
        Assert.assertArrayEquals(new Object[] { 2, "Jane Doe", 4820.09, "321 Your Street", "Your City", "NY", "33333", null }, secondRecordValues);
        final Object[] thirdRecordValues = reader.nextRecord().getValues();
        Assert.assertArrayEquals(new Object[] { 3, "Jake Doe", 4751.89, "124 My Street", "My City", "MS", "11111", "USA" }, thirdRecordValues);
        assertNull(reader.nextRecord());
    }
}

Example 4 with RecordFieldType

use of org.apache.nifi.serialization.record.RecordFieldType in project nifi by apache.

the class TestJsonPathRowRecordReader method testReadArray.

@Test
public void testReadArray() throws IOException, MalformedRecordException {
    final RecordSchema schema = new SimpleRecordSchema(getDefaultFields());
    try (final InputStream in = new FileInputStream(new File("src/test/resources/json/bank-account-array.json"));
        final JsonPathRowRecordReader reader = new JsonPathRowRecordReader(allJsonPaths, schema, in, Mockito.mock(ComponentLog.class), dateFormat, timeFormat, timestampFormat)) {
        final List<String> fieldNames = schema.getFieldNames();
        final List<String> expectedFieldNames = Arrays.asList(new String[] { "id", "name", "balance", "address", "city", "state", "zipCode", "country" });
        assertEquals(expectedFieldNames, fieldNames);
        final List<RecordFieldType> dataTypes = schema.getDataTypes().stream().map(dt -> dt.getFieldType()).collect(Collectors.toList());
        final List<RecordFieldType> expectedTypes = Arrays.asList(new RecordFieldType[] { RecordFieldType.INT, RecordFieldType.STRING, RecordFieldType.DOUBLE, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING });
        assertEquals(expectedTypes, dataTypes);
        final Object[] firstRecordValues = reader.nextRecord().getValues();
        Assert.assertArrayEquals(new Object[] { 1, "John Doe", 4750.89, "123 My Street", "My City", "MS", "11111", "USA" }, firstRecordValues);
        final Object[] secondRecordValues = reader.nextRecord().getValues();
        Assert.assertArrayEquals(new Object[] { 2, "Jane Doe", 4820.09, "321 Your Street", "Your City", "NY", "33333", "USA" }, secondRecordValues);
        assertNull(reader.nextRecord());
    }
}

Example 5 with RecordFieldType

use of org.apache.nifi.serialization.record.RecordFieldType in project nifi by apache.

the class TestJsonPathRowRecordReader method testPrimitiveTypeArrays.

@Test
public void testPrimitiveTypeArrays() throws IOException, MalformedRecordException {
    final LinkedHashMap<String, JsonPath> jsonPaths = new LinkedHashMap<>(allJsonPaths);
    jsonPaths.put("accountIds", JsonPath.compile("$.accountIds"));
    final List<RecordField> fields = getDefaultFields();
    final DataType idsType = RecordFieldType.ARRAY.getArrayDataType(RecordFieldType.INT.getDataType());
    fields.add(new RecordField("accountIds", idsType));
    final RecordSchema schema = new SimpleRecordSchema(fields);
    try (final InputStream in = new FileInputStream(new File("src/test/resources/json/primitive-type-array.json"));
        final JsonPathRowRecordReader reader = new JsonPathRowRecordReader(jsonPaths, schema, in, Mockito.mock(ComponentLog.class), dateFormat, timeFormat, timestampFormat)) {
        final List<String> fieldNames = schema.getFieldNames();
        final List<String> expectedFieldNames = Arrays.asList(new String[] { "id", "name", "balance", "address", "city", "state", "zipCode", "country", "accountIds" });
        assertEquals(expectedFieldNames, fieldNames);
        final List<RecordFieldType> dataTypes = schema.getDataTypes().stream().map(dt -> dt.getFieldType()).collect(Collectors.toList());
        final List<RecordFieldType> expectedTypes = Arrays.asList(new RecordFieldType[] { RecordFieldType.INT, RecordFieldType.STRING, RecordFieldType.DOUBLE, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.ARRAY });
        assertEquals(expectedTypes, dataTypes);
        final Object[] firstRecordValues = reader.nextRecord().getValues();
        final Object[] nonArrayValues = Arrays.copyOfRange(firstRecordValues, 0, firstRecordValues.length - 1);
        Assert.assertArrayEquals(new Object[] { 1, "John Doe", 4750.89D, "123 My Street", "My City", "MS", "11111", "USA" }, nonArrayValues);
        final Object lastRecord = firstRecordValues[firstRecordValues.length - 1];
        assertNotNull(lastRecord);
        assertTrue(Object[].class.isAssignableFrom(lastRecord.getClass()));
        final Object[] array = (Object[]) lastRecord;
        Assert.assertArrayEquals(new Object[] { 1, 2, 3 }, array);
        assertNull(reader.nextRecord());
        assertNull(reader.nextRecord());
    }
}