Examples with ArrayDataType org.apache.nifi.serialization.record.type.ArrayDataType used on opensource projects

Example 6 with ArrayDataType

use of org.apache.nifi.serialization.record.type.ArrayDataType in project nifi by apache.

the class ArrayIndexPath method evaluate.

@Override
public Stream<FieldValue> evaluate(final RecordPathEvaluationContext context) {
    final Stream<FieldValue> parentResult = getParentPath().evaluate(context);
    return parentResult.filter(Filters.fieldTypeFilter(RecordFieldType.ARRAY)).filter(fieldValue -> fieldValue.getValue() != null && ((Object[]) fieldValue.getValue()).length > getArrayIndex(((Object[]) fieldValue.getValue()).length)).map(fieldValue -> {
        final ArrayDataType arrayDataType = (ArrayDataType) fieldValue.getField().getDataType();
        final DataType elementDataType = arrayDataType.getElementType();
        final RecordField arrayField = new RecordField(fieldValue.getField().getFieldName(), elementDataType);
        final Object[] values = (Object[]) fieldValue.getValue();
        final int arrayIndex = getArrayIndex(values.length);
        final RecordField elementField = new RecordField(arrayField.getFieldName(), elementDataType);
        final FieldValue result = new ArrayIndexFieldValue(values[arrayIndex], elementField, fieldValue, arrayIndex);
        return result;
    });
}

Example 7 with ArrayDataType

use of org.apache.nifi.serialization.record.type.ArrayDataType in project nifi by apache.

the class AvroTypeUtil method buildAvroSchema.

private static Schema buildAvroSchema(final DataType dataType, final String fieldName, final boolean nullable) {
    final Schema schema;
    switch(dataType.getFieldType()) {
        case ARRAY:
            final ArrayDataType arrayDataType = (ArrayDataType) dataType;
            final DataType elementDataType = arrayDataType.getElementType();
            if (RecordFieldType.BYTE.equals(elementDataType.getFieldType())) {
                schema = Schema.create(Type.BYTES);
            } else {
                final Schema elementType = buildAvroSchema(elementDataType, fieldName, false);
                schema = Schema.createArray(elementType);
            }
            break;
        case BIGINT:
            schema = Schema.create(Type.STRING);
            break;
        case BOOLEAN:
            schema = Schema.create(Type.BOOLEAN);
            break;
        case BYTE:
            schema = Schema.create(Type.INT);
            break;
        case CHAR:
            schema = Schema.create(Type.STRING);
            break;
        case CHOICE:
            final ChoiceDataType choiceDataType = (ChoiceDataType) dataType;
            final List<DataType> options = choiceDataType.getPossibleSubTypes();
            // We need to keep track of which types have been added to the union, because if we have
            // two elements in the UNION with the same type, it will fail - even if the logical type is
            // different. So if we have an int and a logical type date (which also has a 'concrete type' of int)
            // then an Exception will be thrown when we try to create the union. To avoid this, we just keep track
            // of the Types and avoid adding it in such a case.
            final List<Schema> unionTypes = new ArrayList<>(options.size());
            final Set<Type> typesAdded = new HashSet<>();
            for (final DataType option : options) {
                final Schema optionSchema = buildAvroSchema(option, fieldName, false);
                if (!typesAdded.contains(optionSchema.getType())) {
                    unionTypes.add(optionSchema);
                    typesAdded.add(optionSchema.getType());
                }
            }
            schema = Schema.createUnion(unionTypes);
            break;
        case DATE:
            schema = Schema.create(Type.INT);
            LogicalTypes.date().addToSchema(schema);
            break;
        case DOUBLE:
            schema = Schema.create(Type.DOUBLE);
            break;
        case FLOAT:
            schema = Schema.create(Type.FLOAT);
            break;
        case INT:
            schema = Schema.create(Type.INT);
            break;
        case LONG:
            schema = Schema.create(Type.LONG);
            break;
        case MAP:
            schema = Schema.createMap(buildAvroSchema(((MapDataType) dataType).getValueType(), fieldName, false));
            break;
        case RECORD:
            final RecordDataType recordDataType = (RecordDataType) dataType;
            final RecordSchema childSchema = recordDataType.getChildSchema();
            final List<Field> childFields = new ArrayList<>(childSchema.getFieldCount());
            for (final RecordField field : childSchema.getFields()) {
                childFields.add(buildAvroField(field));
            }
            schema = Schema.createRecord(fieldName + "Type", null, "org.apache.nifi", false, childFields);
            break;
        case SHORT:
            schema = Schema.create(Type.INT);
            break;
        case STRING:
            schema = Schema.create(Type.STRING);
            break;
        case TIME:
            schema = Schema.create(Type.INT);
            LogicalTypes.timeMillis().addToSchema(schema);
            break;
        case TIMESTAMP:
            schema = Schema.create(Type.LONG);
            LogicalTypes.timestampMillis().addToSchema(schema);
            break;
        default:
            return null;
    }
    if (nullable) {
        return nullable(schema);
    } else {
        return schema;
    }
}

Example 8 with ArrayDataType

use of org.apache.nifi.serialization.record.type.ArrayDataType in project nifi by apache.

the class StandardSchemaValidator method isTypeCorrect.

private boolean isTypeCorrect(final Object value, final DataType dataType) {
    switch(dataType.getFieldType()) {
        case ARRAY:
            if (!(value instanceof Object[])) {
                return false;
            }
            final ArrayDataType arrayDataType = (ArrayDataType) dataType;
            final DataType elementType = arrayDataType.getElementType();
            final Object[] array = (Object[]) value;
            for (final Object arrayVal : array) {
                if (!isTypeCorrect(arrayVal, elementType)) {
                    return false;
                }
            }
            return true;
        case MAP:
            if (!(value instanceof Map)) {
                return false;
            }
            final MapDataType mapDataType = (MapDataType) dataType;
            final DataType valueDataType = mapDataType.getValueType();
            final Map<?, ?> map = (Map<?, ?>) value;
            for (final Object mapValue : map.values()) {
                if (!isTypeCorrect(mapValue, valueDataType)) {
                    return false;
                }
            }
            return true;
        case RECORD:
            return value instanceof Record;
        case CHOICE:
            final ChoiceDataType choiceDataType = (ChoiceDataType) dataType;
            for (final DataType choice : choiceDataType.getPossibleSubTypes()) {
                if (isTypeCorrect(value, choice)) {
                    return true;
                }
            }
            return false;
        case BIGINT:
            return value instanceof BigInteger;
        case BOOLEAN:
            return value instanceof Boolean;
        case BYTE:
            return value instanceof Byte;
        case CHAR:
            return value instanceof Character;
        case DATE:
            return value instanceof java.sql.Date;
        case DOUBLE:
            return value instanceof Double;
        case FLOAT:
            // Same goes for Short/Integer
            return value instanceof Float;
        case INT:
            return value instanceof Integer;
        case LONG:
            return value instanceof Long;
        case SHORT:
            return value instanceof Short;
        case STRING:
            return value instanceof String;
        case TIME:
            return value instanceof java.sql.Time;
        case TIMESTAMP:
            return value instanceof java.sql.Timestamp;
    }
    return false;
}

Example 9 with ArrayDataType

use of org.apache.nifi.serialization.record.type.ArrayDataType in project nifi by apache.

the class StandardSchemaValidator method verifyComplexType.

private void verifyComplexType(final DataType dataType, final Object rawValue, final StandardSchemaValidationResult result, final String fieldPrefix, final RecordField field) {
    // If the field type is RECORD, or if the field type is a CHOICE that allows for a RECORD and the value is a RECORD, then we
    // need to dig into each of the sub-fields. To do this, we first need to determine the 'canonical data type'.
    final DataType canonicalDataType = getCanonicalDataType(dataType, rawValue, result, fieldPrefix, field);
    if (canonicalDataType == null) {
        return;
    }
    // Now that we have the 'canonical data type', we check if it is a Record. If so, we need to validate each sub-field.
    if (canonicalDataType.getFieldType() == RecordFieldType.RECORD) {
        verifyChildRecord(canonicalDataType, rawValue, dataType, result, field, fieldPrefix);
    }
    if (canonicalDataType.getFieldType() == RecordFieldType.ARRAY) {
        final ArrayDataType arrayDataType = (ArrayDataType) canonicalDataType;
        final DataType elementType = arrayDataType.getElementType();
        final Object[] arrayObject = (Object[]) rawValue;
        int i = 0;
        for (final Object arrayValue : arrayObject) {
            verifyComplexType(elementType, arrayValue, result, fieldPrefix + "[" + i + "]", field);
            i++;
        }
    }
}

Example 10 with ArrayDataType

use of org.apache.nifi.serialization.record.type.ArrayDataType in project nifi by apache.

the class MultiArrayIndexPath method evaluate.

@Override
public Stream<FieldValue> evaluate(final RecordPathEvaluationContext context) {
    final Stream<FieldValue> parentResult = getParentPath().evaluate(context);
    return parentResult.filter(Filters.fieldTypeFilter(RecordFieldType.ARRAY)).flatMap(fieldValue -> {
        final ArrayDataType arrayDataType = (ArrayDataType) fieldValue.getField().getDataType();
        final DataType elementDataType = arrayDataType.getElementType();
        final RecordField arrayField = new RecordField(fieldValue.getField().getFieldName(), elementDataType);
        final Object[] values = (Object[]) fieldValue.getValue();
        return indices.stream().filter(range -> values.length > Math.abs(range.getMin())).flatMap(range -> {
            final int min = range.getMin() < 0 ? values.length + range.getMin() : range.getMin();
            final int max = range.getMax() < 0 ? values.length + range.getMax() : range.getMax();
            final List<FieldValue> indexFieldValues = new ArrayList<>(Math.max(0, max - min));
            for (int i = min; i <= max; i++) {
                if (values.length > i) {
                    final RecordField elementField = new RecordField(arrayField.getFieldName(), elementDataType);
                    final FieldValue arrayIndexFieldValue = new ArrayIndexFieldValue(values[i], elementField, fieldValue, i);
                    indexFieldValues.add(arrayIndexFieldValue);
                }
            }
            return indexFieldValues.stream();
        });
    });
}