Examples with SimpleRecordSchema org.apache.nifi.serialization.SimpleRecordSchema used on opensource projects

Example 96 with SimpleRecordSchema

use of org.apache.nifi.serialization.SimpleRecordSchema in project nifi by apache.

the class TestStandardSchemaValidator method testInvalidArrayValue.

@Test
public void testInvalidArrayValue() {
    final List<RecordField> fields = new ArrayList<>();
    fields.add(new RecordField("id", RecordFieldType.INT.getDataType()));
    fields.add(new RecordField("numbers", RecordFieldType.ARRAY.getArrayDataType(RecordFieldType.INT.getDataType())));
    final RecordSchema schema = new SimpleRecordSchema(fields);
    final Map<String, Object> valueMap = new LinkedHashMap<>();
    valueMap.put("id", 1);
    valueMap.put("numbers", new Object[] { 1, "2", "3" });
    Record record = new MapRecord(schema, valueMap, false, false);
    final SchemaValidationContext strictValidationContext = new SchemaValidationContext(schema, false, true);
    final SchemaValidationContext lenientValidationContext = new SchemaValidationContext(schema, false, false);
    StandardSchemaValidator validator = new StandardSchemaValidator(strictValidationContext);
    SchemaValidationResult result = validator.validate(record);
    assertFalse(result.isValid());
    assertNotNull(result.getValidationErrors());
    assertEquals(1, result.getValidationErrors().size());
    final ValidationError validationError = result.getValidationErrors().iterator().next();
    assertEquals("/numbers", validationError.getFieldName().get());
    validator = new StandardSchemaValidator(lenientValidationContext);
    result = validator.validate(record);
    assertTrue(result.isValid());
    assertNotNull(result.getValidationErrors());
    assertTrue(result.getValidationErrors().isEmpty());
}

Example 97 with SimpleRecordSchema

use of org.apache.nifi.serialization.SimpleRecordSchema in project nifi by apache.

the class TestStandardSchemaValidator method testMissingRequiredField.

@Test
public void testMissingRequiredField() {
    final List<RecordField> fields = new ArrayList<>();
    fields.add(new RecordField("id", RecordFieldType.INT.getDataType()));
    fields.add(new RecordField("name", RecordFieldType.STRING.getDataType(), false));
    final RecordSchema schema = new SimpleRecordSchema(fields);
    final Map<String, Object> valueMap = new LinkedHashMap<>();
    valueMap.put("id", 1);
    final Record record = new MapRecord(schema, valueMap, false, false);
    final SchemaValidationContext allowExtraFieldsContext = new SchemaValidationContext(schema, true, true);
    StandardSchemaValidator validator = new StandardSchemaValidator(allowExtraFieldsContext);
    SchemaValidationResult result = validator.validate(record);
    assertFalse(result.isValid());
    assertNotNull(result.getValidationErrors());
    final ValidationError error = result.getValidationErrors().iterator().next();
    assertEquals("/name", error.getFieldName().get());
}

Example 98 with SimpleRecordSchema

use of org.apache.nifi.serialization.SimpleRecordSchema in project nifi by apache.

the class PutElasticsearchHttpRecordIT method setupTests.

@BeforeClass
public static void setupTests() throws Exception {
    final List<RecordField> personFields = new ArrayList<>();
    final RecordField nameField = new RecordField("name", RecordFieldType.STRING.getDataType());
    final RecordField ageField = new RecordField("age", RecordFieldType.INT.getDataType());
    final RecordField sportField = new RecordField("sport", RecordFieldType.STRING.getDataType());
    personFields.add(nameField);
    personFields.add(ageField);
    personFields.add(sportField);
    personSchema = new SimpleRecordSchema(personFields);
    FETCH_RUNNER = TestRunners.newTestRunner(FetchElasticsearchHttp.class);
    FETCH_RUNNER.setProperty(FetchElasticsearchHttp.ES_URL, "http://localhost:9200");
    FETCH_RUNNER.setProperty(FetchElasticsearchHttp.INDEX, "people_test");
    FETCH_RUNNER.setProperty(FetchElasticsearchHttp.TYPE, "person");
    FETCH_RUNNER.setProperty(FetchElasticsearchHttp.DOC_ID, "${doc_id}");
    FETCH_RUNNER.assertValid();
}

Example 99 with SimpleRecordSchema

use of org.apache.nifi.serialization.SimpleRecordSchema in project nifi by apache.

the class AvroTypeUtil method determineDataType.

public static DataType determineDataType(final Schema avroSchema, Map<String, DataType> knownRecordTypes) {
    if (knownRecordTypes == null) {
        throw new IllegalArgumentException("'knownRecordTypes' cannot be null.");
    }
    final Type avroType = avroSchema.getType();
    final LogicalType logicalType = avroSchema.getLogicalType();
    if (logicalType != null) {
        final String logicalTypeName = logicalType.getName();
        switch(logicalTypeName) {
            case LOGICAL_TYPE_DATE:
                return RecordFieldType.DATE.getDataType();
            case LOGICAL_TYPE_TIME_MILLIS:
            case LOGICAL_TYPE_TIME_MICROS:
                return RecordFieldType.TIME.getDataType();
            case LOGICAL_TYPE_TIMESTAMP_MILLIS:
            case LOGICAL_TYPE_TIMESTAMP_MICROS:
                return RecordFieldType.TIMESTAMP.getDataType();
            case LOGICAL_TYPE_DECIMAL:
                // Alternatively we could convert it to String, but numeric type is generally more preferable by users.
                return RecordFieldType.DOUBLE.getDataType();
        }
    }
    switch(avroType) {
        case ARRAY:
            return RecordFieldType.ARRAY.getArrayDataType(determineDataType(avroSchema.getElementType(), knownRecordTypes));
        case BYTES:
        case FIXED:
            return RecordFieldType.ARRAY.getArrayDataType(RecordFieldType.BYTE.getDataType());
        case BOOLEAN:
            return RecordFieldType.BOOLEAN.getDataType();
        case DOUBLE:
            return RecordFieldType.DOUBLE.getDataType();
        case ENUM:
        case STRING:
            return RecordFieldType.STRING.getDataType();
        case FLOAT:
            return RecordFieldType.FLOAT.getDataType();
        case INT:
            return RecordFieldType.INT.getDataType();
        case LONG:
            return RecordFieldType.LONG.getDataType();
        case RECORD:
            {
                String schemaFullName = avroSchema.getNamespace() + "." + avroSchema.getName();
                if (knownRecordTypes.containsKey(schemaFullName)) {
                    return knownRecordTypes.get(schemaFullName);
                } else {
                    SimpleRecordSchema recordSchema = new SimpleRecordSchema(avroSchema.toString(), AVRO_SCHEMA_FORMAT, SchemaIdentifier.EMPTY);
                    DataType recordSchemaType = RecordFieldType.RECORD.getRecordDataType(recordSchema);
                    knownRecordTypes.put(schemaFullName, recordSchemaType);
                    final List<Field> avroFields = avroSchema.getFields();
                    final List<RecordField> recordFields = new ArrayList<>(avroFields.size());
                    for (final Field field : avroFields) {
                        final String fieldName = field.name();
                        final Schema fieldSchema = field.schema();
                        final DataType fieldType = determineDataType(fieldSchema, knownRecordTypes);
                        final boolean nullable = isNullable(fieldSchema);
                        addFieldToList(recordFields, field, fieldName, fieldSchema, fieldType, nullable);
                    }
                    recordSchema.setFields(recordFields);
                    return recordSchemaType;
                }
            }
        case NULL:
            return RecordFieldType.STRING.getDataType();
        case MAP:
            final Schema valueSchema = avroSchema.getValueType();
            final DataType valueType = determineDataType(valueSchema, knownRecordTypes);
            return RecordFieldType.MAP.getMapDataType(valueType);
        case UNION:
            {
                final List<Schema> nonNullSubSchemas = getNonNullSubSchemas(avroSchema);
                if (nonNullSubSchemas.size() == 1) {
                    return determineDataType(nonNullSubSchemas.get(0), knownRecordTypes);
                }
                final List<DataType> possibleChildTypes = new ArrayList<>(nonNullSubSchemas.size());
                for (final Schema subSchema : nonNullSubSchemas) {
                    final DataType childDataType = determineDataType(subSchema, knownRecordTypes);
                    possibleChildTypes.add(childDataType);
                }
                return RecordFieldType.CHOICE.getChoiceDataType(possibleChildTypes);
            }
    }
    return null;
}

Example 100 with SimpleRecordSchema

use of org.apache.nifi.serialization.SimpleRecordSchema in project nifi by apache.

the class CommaSeparatedRecordReader method createRecordReader.

@Override
public RecordReader createRecordReader(Map<String, String> variables, final InputStream in, final ComponentLog logger) throws IOException, SchemaNotFoundException {
    final BufferedReader reader = new BufferedReader(new InputStreamReader(in));
    final List<RecordField> fields = new ArrayList<>();
    final String headerLine = reader.readLine();
    for (final String colName : headerLine.split(",")) {
        fields.add(new RecordField(colName.trim(), RecordFieldType.STRING.getDataType()));
    }
    return new RecordReader() {

        @Override
        public void close() throws IOException {
            reader.close();
        }

        @Override
        public Record nextRecord(final boolean coerceTypes, final boolean dropUnknown) throws IOException, MalformedRecordException {
            if (failAfterN > -1 && recordCount >= failAfterN) {
                throw new MalformedRecordException("Intentional Unit Test Exception because " + recordCount + " records have been read");
            }
            final String nextLine = reader.readLine();
            if (nextLine == null) {
                return null;
            }
            recordCount++;
            final String[] values = nextLine.split(",");
            final Map<String, Object> valueMap = new HashMap<>();
            int i = 0;
            for (final RecordField field : fields) {
                final String fieldName = field.getFieldName();
                valueMap.put(fieldName, values[i++].trim());
            }
            return new MapRecord(new SimpleRecordSchema(fields), valueMap);
        }

        @Override
        public RecordSchema getSchema() {
            return new SimpleRecordSchema(fields);
        }
    };
}