Search in sources :

Example 11 with MalformedRecordException

use of org.apache.nifi.serialization.MalformedRecordException in project nifi by apache.

the class AvroReader method createRecordReader.

@Override
public RecordReader createRecordReader(final Map<String, String> variables, final InputStream in, final ComponentLog logger) throws MalformedRecordException, IOException, SchemaNotFoundException {
    final String schemaAccessStrategy = getConfigurationContext().getProperty(getSchemaAcessStrategyDescriptor()).getValue();
    if (EMBEDDED_AVRO_SCHEMA.getValue().equals(schemaAccessStrategy)) {
        return new AvroReaderWithEmbeddedSchema(in);
    } else {
        final RecordSchema recordSchema = getSchema(variables, in, null);
        final Schema avroSchema;
        try {
            if (recordSchema.getSchemaFormat().isPresent() & recordSchema.getSchemaFormat().get().equals(AvroTypeUtil.AVRO_SCHEMA_FORMAT)) {
                final Optional<String> textOption = recordSchema.getSchemaText();
                if (textOption.isPresent()) {
                    avroSchema = compileAvroSchema(textOption.get());
                } else {
                    avroSchema = AvroTypeUtil.extractAvroSchema(recordSchema);
                }
            } else {
                avroSchema = AvroTypeUtil.extractAvroSchema(recordSchema);
            }
        } catch (final Exception e) {
            throw new SchemaNotFoundException("Failed to compile Avro Schema", e);
        }
        return new AvroReaderWithExplicitSchema(in, recordSchema, avroSchema);
    }
}
Also used : Schema(org.apache.avro.Schema) RecordSchema(org.apache.nifi.serialization.record.RecordSchema) SchemaNotFoundException(org.apache.nifi.schema.access.SchemaNotFoundException) RecordSchema(org.apache.nifi.serialization.record.RecordSchema) MalformedRecordException(org.apache.nifi.serialization.MalformedRecordException) IOException(java.io.IOException) SchemaNotFoundException(org.apache.nifi.schema.access.SchemaNotFoundException)

Example 12 with MalformedRecordException

use of org.apache.nifi.serialization.MalformedRecordException in project nifi by apache.

the class AbstractJsonRowRecordReader method nextRecord.

@Override
public Record nextRecord(final boolean coerceTypes, final boolean dropUnknownFields) throws IOException, MalformedRecordException {
    if (firstObjectConsumed && !array) {
        return null;
    }
    final JsonNode nextNode = getNextJsonNode();
    final RecordSchema schema = getSchema();
    try {
        return convertJsonNodeToRecord(nextNode, schema, coerceTypes, dropUnknownFields);
    } catch (final MalformedRecordException mre) {
        throw mre;
    } catch (final IOException ioe) {
        throw ioe;
    } catch (final Exception e) {
        logger.debug("Failed to convert JSON Element {} into a Record object using schema {} due to {}", new Object[] { nextNode, schema, e.toString(), e });
        throw new MalformedRecordException("Successfully parsed a JSON object from input but failed to convert into a Record object with the given schema", e);
    }
}
Also used : JsonNode(org.codehaus.jackson.JsonNode) IOException(java.io.IOException) RecordSchema(org.apache.nifi.serialization.record.RecordSchema) SimpleRecordSchema(org.apache.nifi.serialization.SimpleRecordSchema) MalformedRecordException(org.apache.nifi.serialization.MalformedRecordException) IOException(java.io.IOException) JsonParseException(org.codehaus.jackson.JsonParseException) MalformedRecordException(org.apache.nifi.serialization.MalformedRecordException)

Example 13 with MalformedRecordException

use of org.apache.nifi.serialization.MalformedRecordException in project nifi by apache.

the class TestJsonPathRowRecordReader method testElementWithNestedArray.

@Test
public void testElementWithNestedArray() throws IOException, MalformedRecordException {
    final LinkedHashMap<String, JsonPath> jsonPaths = new LinkedHashMap<>(allJsonPaths);
    jsonPaths.put("accounts", JsonPath.compile("$.accounts"));
    final DataType accountRecordType = RecordFieldType.RECORD.getRecordDataType(getAccountSchema());
    final DataType accountsType = RecordFieldType.ARRAY.getArrayDataType(accountRecordType);
    final List<RecordField> fields = getDefaultFields();
    fields.add(new RecordField("accounts", accountsType));
    final RecordSchema schema = new SimpleRecordSchema(fields);
    try (final InputStream in = new FileInputStream(new File("src/test/resources/json/single-element-nested-array.json"));
        final JsonPathRowRecordReader reader = new JsonPathRowRecordReader(jsonPaths, schema, in, Mockito.mock(ComponentLog.class), dateFormat, timeFormat, timestampFormat)) {
        final List<String> fieldNames = schema.getFieldNames();
        final List<String> expectedFieldNames = Arrays.asList(new String[] { "id", "name", "balance", "address", "city", "state", "zipCode", "country", "accounts" });
        assertEquals(expectedFieldNames, fieldNames);
        final List<RecordFieldType> dataTypes = schema.getDataTypes().stream().map(dt -> dt.getFieldType()).collect(Collectors.toList());
        final List<RecordFieldType> expectedTypes = Arrays.asList(new RecordFieldType[] { RecordFieldType.INT, RecordFieldType.STRING, RecordFieldType.DOUBLE, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.ARRAY });
        assertEquals(expectedTypes, dataTypes);
        final Object[] firstRecordValues = reader.nextRecord().getValues();
        final Object[] nonArrayValues = Arrays.copyOfRange(firstRecordValues, 0, firstRecordValues.length - 1);
        Assert.assertArrayEquals(new Object[] { 1, "John Doe", null, "123 My Street", "My City", "MS", "11111", "USA" }, nonArrayValues);
        final Object lastRecord = firstRecordValues[firstRecordValues.length - 1];
        assertTrue(Object[].class.isAssignableFrom(lastRecord.getClass()));
        final Object[] array = (Object[]) lastRecord;
        assertEquals(2, array.length);
        final Object firstElement = array[0];
        assertTrue(firstElement instanceof Record);
        final Record firstRecord = (Record) firstElement;
        assertEquals(42, firstRecord.getValue("id"));
        assertEquals(4750.89D, firstRecord.getValue("balance"));
        final Object secondElement = array[1];
        assertTrue(secondElement instanceof Record);
        final Record secondRecord = (Record) secondElement;
        assertEquals(43, secondRecord.getValue("id"));
        assertEquals(48212.38D, secondRecord.getValue("balance"));
        assertNull(reader.nextRecord());
    }
}
Also used : SimpleRecordSchema(org.apache.nifi.serialization.SimpleRecordSchema) Arrays(java.util.Arrays) DataType(org.apache.nifi.serialization.record.DataType) ComponentLog(org.apache.nifi.logging.ComponentLog) ArrayList(java.util.ArrayList) LinkedHashMap(java.util.LinkedHashMap) RecordSchema(org.apache.nifi.serialization.record.RecordSchema) Record(org.apache.nifi.serialization.record.Record) Before(org.junit.Before) MalformedRecordException(org.apache.nifi.serialization.MalformedRecordException) Assert.assertNotNull(org.junit.Assert.assertNotNull) RecordField(org.apache.nifi.serialization.record.RecordField) Assert.assertTrue(org.junit.Assert.assertTrue) IOException(java.io.IOException) SimpleRecordSchema(org.apache.nifi.serialization.SimpleRecordSchema) Test(org.junit.Test) FileInputStream(java.io.FileInputStream) JsonPath(com.jayway.jsonpath.JsonPath) Collectors(java.util.stream.Collectors) File(java.io.File) Mockito(org.mockito.Mockito) List(java.util.List) Assert.assertNull(org.junit.Assert.assertNull) Assert(org.junit.Assert) RecordFieldType(org.apache.nifi.serialization.record.RecordFieldType) Assert.assertEquals(org.junit.Assert.assertEquals) InputStream(java.io.InputStream) RecordField(org.apache.nifi.serialization.record.RecordField) FileInputStream(java.io.FileInputStream) InputStream(java.io.InputStream) JsonPath(com.jayway.jsonpath.JsonPath) ComponentLog(org.apache.nifi.logging.ComponentLog) FileInputStream(java.io.FileInputStream) LinkedHashMap(java.util.LinkedHashMap) DataType(org.apache.nifi.serialization.record.DataType) Record(org.apache.nifi.serialization.record.Record) RecordSchema(org.apache.nifi.serialization.record.RecordSchema) SimpleRecordSchema(org.apache.nifi.serialization.SimpleRecordSchema) File(java.io.File) RecordFieldType(org.apache.nifi.serialization.record.RecordFieldType) Test(org.junit.Test)

Example 14 with MalformedRecordException

use of org.apache.nifi.serialization.MalformedRecordException in project nifi by apache.

the class TestJsonPathRowRecordReader method testReadArrayDifferentSchemas.

@Test
public void testReadArrayDifferentSchemas() throws IOException, MalformedRecordException {
    final RecordSchema schema = new SimpleRecordSchema(getDefaultFields());
    try (final InputStream in = new FileInputStream(new File("src/test/resources/json/bank-account-array-different-schemas.json"));
        final JsonPathRowRecordReader reader = new JsonPathRowRecordReader(allJsonPaths, schema, in, Mockito.mock(ComponentLog.class), dateFormat, timeFormat, timestampFormat)) {
        final List<String> fieldNames = schema.getFieldNames();
        final List<String> expectedFieldNames = Arrays.asList(new String[] { "id", "name", "balance", "address", "city", "state", "zipCode", "country" });
        assertEquals(expectedFieldNames, fieldNames);
        final List<RecordFieldType> dataTypes = schema.getDataTypes().stream().map(dt -> dt.getFieldType()).collect(Collectors.toList());
        final List<RecordFieldType> expectedTypes = Arrays.asList(new RecordFieldType[] { RecordFieldType.INT, RecordFieldType.STRING, RecordFieldType.DOUBLE, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING });
        assertEquals(expectedTypes, dataTypes);
        final Object[] firstRecordValues = reader.nextRecord().getValues();
        Assert.assertArrayEquals(new Object[] { 1, "John Doe", 4750.89, "123 My Street", "My City", "MS", "11111", "USA" }, firstRecordValues);
        final Object[] secondRecordValues = reader.nextRecord().getValues();
        Assert.assertArrayEquals(new Object[] { 2, "Jane Doe", 4820.09, "321 Your Street", "Your City", "NY", "33333", null }, secondRecordValues);
        final Object[] thirdRecordValues = reader.nextRecord().getValues();
        Assert.assertArrayEquals(new Object[] { 3, "Jake Doe", 4751.89, "124 My Street", "My City", "MS", "11111", "USA" }, thirdRecordValues);
        assertNull(reader.nextRecord());
    }
}
Also used : SimpleRecordSchema(org.apache.nifi.serialization.SimpleRecordSchema) Arrays(java.util.Arrays) DataType(org.apache.nifi.serialization.record.DataType) ComponentLog(org.apache.nifi.logging.ComponentLog) ArrayList(java.util.ArrayList) LinkedHashMap(java.util.LinkedHashMap) RecordSchema(org.apache.nifi.serialization.record.RecordSchema) Record(org.apache.nifi.serialization.record.Record) Before(org.junit.Before) MalformedRecordException(org.apache.nifi.serialization.MalformedRecordException) Assert.assertNotNull(org.junit.Assert.assertNotNull) RecordField(org.apache.nifi.serialization.record.RecordField) Assert.assertTrue(org.junit.Assert.assertTrue) IOException(java.io.IOException) SimpleRecordSchema(org.apache.nifi.serialization.SimpleRecordSchema) Test(org.junit.Test) FileInputStream(java.io.FileInputStream) JsonPath(com.jayway.jsonpath.JsonPath) Collectors(java.util.stream.Collectors) File(java.io.File) Mockito(org.mockito.Mockito) List(java.util.List) Assert.assertNull(org.junit.Assert.assertNull) Assert(org.junit.Assert) RecordFieldType(org.apache.nifi.serialization.record.RecordFieldType) Assert.assertEquals(org.junit.Assert.assertEquals) InputStream(java.io.InputStream) FileInputStream(java.io.FileInputStream) InputStream(java.io.InputStream) ComponentLog(org.apache.nifi.logging.ComponentLog) FileInputStream(java.io.FileInputStream) RecordSchema(org.apache.nifi.serialization.record.RecordSchema) SimpleRecordSchema(org.apache.nifi.serialization.SimpleRecordSchema) File(java.io.File) RecordFieldType(org.apache.nifi.serialization.record.RecordFieldType) Test(org.junit.Test)

Example 15 with MalformedRecordException

use of org.apache.nifi.serialization.MalformedRecordException in project nifi by apache.

the class TestJsonPathRowRecordReader method testReadArray.

@Test
public void testReadArray() throws IOException, MalformedRecordException {
    final RecordSchema schema = new SimpleRecordSchema(getDefaultFields());
    try (final InputStream in = new FileInputStream(new File("src/test/resources/json/bank-account-array.json"));
        final JsonPathRowRecordReader reader = new JsonPathRowRecordReader(allJsonPaths, schema, in, Mockito.mock(ComponentLog.class), dateFormat, timeFormat, timestampFormat)) {
        final List<String> fieldNames = schema.getFieldNames();
        final List<String> expectedFieldNames = Arrays.asList(new String[] { "id", "name", "balance", "address", "city", "state", "zipCode", "country" });
        assertEquals(expectedFieldNames, fieldNames);
        final List<RecordFieldType> dataTypes = schema.getDataTypes().stream().map(dt -> dt.getFieldType()).collect(Collectors.toList());
        final List<RecordFieldType> expectedTypes = Arrays.asList(new RecordFieldType[] { RecordFieldType.INT, RecordFieldType.STRING, RecordFieldType.DOUBLE, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING });
        assertEquals(expectedTypes, dataTypes);
        final Object[] firstRecordValues = reader.nextRecord().getValues();
        Assert.assertArrayEquals(new Object[] { 1, "John Doe", 4750.89, "123 My Street", "My City", "MS", "11111", "USA" }, firstRecordValues);
        final Object[] secondRecordValues = reader.nextRecord().getValues();
        Assert.assertArrayEquals(new Object[] { 2, "Jane Doe", 4820.09, "321 Your Street", "Your City", "NY", "33333", "USA" }, secondRecordValues);
        assertNull(reader.nextRecord());
    }
}
Also used : SimpleRecordSchema(org.apache.nifi.serialization.SimpleRecordSchema) Arrays(java.util.Arrays) DataType(org.apache.nifi.serialization.record.DataType) ComponentLog(org.apache.nifi.logging.ComponentLog) ArrayList(java.util.ArrayList) LinkedHashMap(java.util.LinkedHashMap) RecordSchema(org.apache.nifi.serialization.record.RecordSchema) Record(org.apache.nifi.serialization.record.Record) Before(org.junit.Before) MalformedRecordException(org.apache.nifi.serialization.MalformedRecordException) Assert.assertNotNull(org.junit.Assert.assertNotNull) RecordField(org.apache.nifi.serialization.record.RecordField) Assert.assertTrue(org.junit.Assert.assertTrue) IOException(java.io.IOException) SimpleRecordSchema(org.apache.nifi.serialization.SimpleRecordSchema) Test(org.junit.Test) FileInputStream(java.io.FileInputStream) JsonPath(com.jayway.jsonpath.JsonPath) Collectors(java.util.stream.Collectors) File(java.io.File) Mockito(org.mockito.Mockito) List(java.util.List) Assert.assertNull(org.junit.Assert.assertNull) Assert(org.junit.Assert) RecordFieldType(org.apache.nifi.serialization.record.RecordFieldType) Assert.assertEquals(org.junit.Assert.assertEquals) InputStream(java.io.InputStream) FileInputStream(java.io.FileInputStream) InputStream(java.io.InputStream) ComponentLog(org.apache.nifi.logging.ComponentLog) FileInputStream(java.io.FileInputStream) RecordSchema(org.apache.nifi.serialization.record.RecordSchema) SimpleRecordSchema(org.apache.nifi.serialization.SimpleRecordSchema) File(java.io.File) RecordFieldType(org.apache.nifi.serialization.record.RecordFieldType) Test(org.junit.Test)

Aggregations

MalformedRecordException (org.apache.nifi.serialization.MalformedRecordException)39 IOException (java.io.IOException)30 InputStream (java.io.InputStream)28 RecordSchema (org.apache.nifi.serialization.record.RecordSchema)28 Record (org.apache.nifi.serialization.record.Record)24 SimpleRecordSchema (org.apache.nifi.serialization.SimpleRecordSchema)21 ComponentLog (org.apache.nifi.logging.ComponentLog)20 RecordField (org.apache.nifi.serialization.record.RecordField)20 ArrayList (java.util.ArrayList)19 Test (org.junit.Test)19 FileInputStream (java.io.FileInputStream)17 File (java.io.File)16 Arrays (java.util.Arrays)16 HashMap (java.util.HashMap)16 List (java.util.List)16 Collectors (java.util.stream.Collectors)16 RecordReader (org.apache.nifi.serialization.RecordReader)16 DataType (org.apache.nifi.serialization.record.DataType)16 RecordFieldType (org.apache.nifi.serialization.record.RecordFieldType)16 Assert.assertEquals (org.junit.Assert.assertEquals)16