Examples with RecordSchema org.apache.nifi.serialization.record.RecordSchema used on opensource projects

Example 56 with RecordSchema

use of org.apache.nifi.serialization.record.RecordSchema in project nifi by apache.

the class TestWriteCSVResult method testExtraFieldInWriteRecord.

@Test
public void testExtraFieldInWriteRecord() throws IOException {
    final CSVFormat csvFormat = CSVFormat.DEFAULT.withEscape('\\').withQuoteMode(QuoteMode.NONE).withRecordSeparator("\n");
    final List<RecordField> fields = new ArrayList<>();
    fields.add(new RecordField("id", RecordFieldType.STRING.getDataType()));
    final RecordSchema schema = new SimpleRecordSchema(fields);
    final Map<String, Object> values = new HashMap<>();
    values.put("id", "1");
    values.put("name", "John");
    final Record record = new MapRecord(schema, values);
    final ByteArrayOutputStream baos = new ByteArrayOutputStream();
    final String output;
    try (final WriteCSVResult writer = new WriteCSVResult(csvFormat, schema, new SchemaNameAsAttribute(), baos, RecordFieldType.DATE.getDefaultFormat(), RecordFieldType.TIME.getDefaultFormat(), RecordFieldType.TIMESTAMP.getDefaultFormat(), true, "ASCII")) {
        writer.beginRecordSet();
        writer.write(record);
        writer.finishRecordSet();
        writer.flush();
        output = baos.toString();
    }
    assertEquals("id\n1\n", output);
}

Example 57 with RecordSchema

use of org.apache.nifi.serialization.record.RecordSchema in project nifi by apache.

the class TestWriteCSVResult method testMissingFieldWriteRawRecord.

@Test
public void testMissingFieldWriteRawRecord() throws IOException {
    final CSVFormat csvFormat = CSVFormat.DEFAULT.withEscape('\\').withQuoteMode(QuoteMode.NONE).withRecordSeparator("\n");
    final List<RecordField> fields = new ArrayList<>();
    fields.add(new RecordField("id", RecordFieldType.STRING.getDataType()));
    fields.add(new RecordField("name", RecordFieldType.STRING.getDataType()));
    final RecordSchema schema = new SimpleRecordSchema(fields);
    final Map<String, Object> values = new LinkedHashMap<>();
    values.put("id", "1");
    final Record record = new MapRecord(schema, values);
    final ByteArrayOutputStream baos = new ByteArrayOutputStream();
    final String output;
    try (final WriteCSVResult writer = new WriteCSVResult(csvFormat, schema, new SchemaNameAsAttribute(), baos, RecordFieldType.DATE.getDefaultFormat(), RecordFieldType.TIME.getDefaultFormat(), RecordFieldType.TIMESTAMP.getDefaultFormat(), true, "ASCII")) {
        writer.beginRecordSet();
        writer.writeRawRecord(record);
        writer.finishRecordSet();
        writer.flush();
        output = baos.toString();
    }
    assertEquals("id,name\n1,\n", output);
}

Example 58 with RecordSchema

use of org.apache.nifi.serialization.record.RecordSchema in project nifi by apache.

the class TestGrokRecordReader method testInheritNamedParameters.

@Test
public void testInheritNamedParameters() throws FileNotFoundException, IOException, GrokException, MalformedRecordException {
    final String syslogMsg = "May 22 15:58:23 my-host nifi[12345]:My Message";
    final byte[] msgBytes = syslogMsg.getBytes();
    try (final InputStream in = new ByteArrayInputStream(msgBytes)) {
        final Grok grok = new Grok();
        grok.addPatternFromFile("src/main/resources/default-grok-patterns.txt");
        grok.compile("%{SYSLOGBASE}%{GREEDYDATA:message}");
        final RecordSchema schema = GrokReader.createRecordSchema(grok);
        final List<String> fieldNames = schema.getFieldNames();
        assertEquals(9, fieldNames.size());
        assertTrue(fieldNames.contains("timestamp"));
        assertTrue(fieldNames.contains("logsource"));
        assertTrue(fieldNames.contains("facility"));
        assertTrue(fieldNames.contains("priority"));
        assertTrue(fieldNames.contains("program"));
        assertTrue(fieldNames.contains("pid"));
        assertTrue(fieldNames.contains("message"));
        // always implicitly there
        assertTrue(fieldNames.contains("stackTrace"));
        // always implicitly there
        assertTrue(fieldNames.contains("_raw"));
        final GrokRecordReader deserializer = new GrokRecordReader(in, grok, schema, schema, true);
        final Record record = deserializer.nextRecord();
        assertEquals("May 22 15:58:23", record.getValue("timestamp"));
        assertEquals("my-host", record.getValue("logsource"));
        assertNull(record.getValue("facility"));
        assertNull(record.getValue("priority"));
        assertEquals("nifi", record.getValue("program"));
        assertEquals("12345", record.getValue("pid"));
        assertEquals("My Message", record.getValue("message"));
        assertEquals("May 22 15:58:23 my-host nifi[12345]:My Message", record.getValue("_raw"));
        assertNull(deserializer.nextRecord());
        deserializer.close();
    }
}

Example 59 with RecordSchema

use of org.apache.nifi.serialization.record.RecordSchema in project nifi by apache.

the class TestJsonPathRowRecordReader method testElementWithNestedArray.

@Test
public void testElementWithNestedArray() throws IOException, MalformedRecordException {
    final LinkedHashMap<String, JsonPath> jsonPaths = new LinkedHashMap<>(allJsonPaths);
    jsonPaths.put("accounts", JsonPath.compile("$.accounts"));
    final DataType accountRecordType = RecordFieldType.RECORD.getRecordDataType(getAccountSchema());
    final DataType accountsType = RecordFieldType.ARRAY.getArrayDataType(accountRecordType);
    final List<RecordField> fields = getDefaultFields();
    fields.add(new RecordField("accounts", accountsType));
    final RecordSchema schema = new SimpleRecordSchema(fields);
    try (final InputStream in = new FileInputStream(new File("src/test/resources/json/single-element-nested-array.json"));
        final JsonPathRowRecordReader reader = new JsonPathRowRecordReader(jsonPaths, schema, in, Mockito.mock(ComponentLog.class), dateFormat, timeFormat, timestampFormat)) {
        final List<String> fieldNames = schema.getFieldNames();
        final List<String> expectedFieldNames = Arrays.asList(new String[] { "id", "name", "balance", "address", "city", "state", "zipCode", "country", "accounts" });
        assertEquals(expectedFieldNames, fieldNames);
        final List<RecordFieldType> dataTypes = schema.getDataTypes().stream().map(dt -> dt.getFieldType()).collect(Collectors.toList());
        final List<RecordFieldType> expectedTypes = Arrays.asList(new RecordFieldType[] { RecordFieldType.INT, RecordFieldType.STRING, RecordFieldType.DOUBLE, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.ARRAY });
        assertEquals(expectedTypes, dataTypes);
        final Object[] firstRecordValues = reader.nextRecord().getValues();
        final Object[] nonArrayValues = Arrays.copyOfRange(firstRecordValues, 0, firstRecordValues.length - 1);
        Assert.assertArrayEquals(new Object[] { 1, "John Doe", null, "123 My Street", "My City", "MS", "11111", "USA" }, nonArrayValues);
        final Object lastRecord = firstRecordValues[firstRecordValues.length - 1];
        assertTrue(Object[].class.isAssignableFrom(lastRecord.getClass()));
        final Object[] array = (Object[]) lastRecord;
        assertEquals(2, array.length);
        final Object firstElement = array[0];
        assertTrue(firstElement instanceof Record);
        final Record firstRecord = (Record) firstElement;
        assertEquals(42, firstRecord.getValue("id"));
        assertEquals(4750.89D, firstRecord.getValue("balance"));
        final Object secondElement = array[1];
        assertTrue(secondElement instanceof Record);
        final Record secondRecord = (Record) secondElement;
        assertEquals(43, secondRecord.getValue("id"));
        assertEquals(48212.38D, secondRecord.getValue("balance"));
        assertNull(reader.nextRecord());
    }
}

Example 60 with RecordSchema

use of org.apache.nifi.serialization.record.RecordSchema in project nifi by apache.

the class TestJsonPathRowRecordReader method getAccountSchema.

private RecordSchema getAccountSchema() {
    final List<RecordField> accountFields = new ArrayList<>();
    accountFields.add(new RecordField("id", RecordFieldType.INT.getDataType()));
    accountFields.add(new RecordField("balance", RecordFieldType.DOUBLE.getDataType()));
    final RecordSchema accountSchema = new SimpleRecordSchema(accountFields);
    return accountSchema;
}