Examples with KafkaConnectField io.atlasmap.kafkaconnect.v2.KafkaConnectField used on opensource projects

Example 6 with KafkaConnectField

use of io.atlasmap.kafkaconnect.v2.KafkaConnectField in project atlasmap by atlasmap.

the class KafkaConnectFieldReader method createValueFields.

private List<Field> createValueFields(Object parent, SegmentContext segment, int segmentIndex, KafkaConnectField parentField) throws AtlasException {
    List<Field> fields = new LinkedList<>();
    if (segment.getCollectionType() == CollectionType.NONE) {
        KafkaConnectField kcField = AtlasKafkaConnectModelFactory.cloneField(parentField, true);
        Object converted = conversionService.convertType(parent, parentField.getFormat(), parentField.getFieldType(), null);
        kcField.setValue(converted);
        // reset index for subfields
        kcField.setIndex(null);
        fields.add(kcField);
    } else if (segment.getCollectionIndex() != null) {
        List<Object> collection = (List<Object>) parent;
        int i = segment.getCollectionIndex();
        KafkaConnectField kcField = AtlasKafkaConnectModelFactory.cloneField(parentField, true);
        Object converted = conversionService.convertType(collection.get(i), parentField.getFormat(), parentField.getFieldType(), null);
        kcField.setValue(converted);
        // reset index for subfields
        kcField.setIndex(null);
        fields.add(kcField);
    } else {
        List<Object> collection = (List<Object>) parent;
        for (int i = 0; i < collection.size(); i++) {
            KafkaConnectField kcField = AtlasKafkaConnectModelFactory.cloneField(parentField, true);
            Object converted = conversionService.convertType(collection.get(i), parentField.getFormat(), parentField.getFieldType(), null);
            kcField.setValue(converted);
            // reset index for subfields
            kcField.setIndex(null);
            fields.add(kcField);
            AtlasPath path = new AtlasPath(parentField.getPath());
            path.setCollectionIndex(segmentIndex, i);
            kcField.setPath(path.toString());
        }
    }
    return fields;
}

Example 7 with KafkaConnectField

use of io.atlasmap.kafkaconnect.v2.KafkaConnectField in project atlasmap by atlasmap.

the class KafkaConnectFieldReader method getFieldsForPath.

private List<Field> getFieldsForPath(AtlasInternalSession session, Object parent, Field field, AtlasPath path, int depth) throws AtlasException {
    List<Field> fields = new ArrayList<>();
    List<SegmentContext> segments = path.getSegments(true);
    if (parent == null) {
        return fields;
    }
    if (segments.size() < depth) {
        throw new AtlasException(String.format("depth '%s' exceeds segment size '%s'", depth, segments.size()));
    }
    if (segments.size() == depth) {
        // if traversed the entire path and found value
        if (field instanceof FieldGroup && field.getFieldType() == FieldType.COMPLEX && (parent instanceof Struct)) {
            FieldGroup group = (FieldGroup) field;
            populateChildFields(session, (Struct) parent, group);
            fields.add(group);
        } else {
            field.setValue(parent);
            fields.add(field);
        }
        return fields;
    }
    // segments.size() > depth
    SegmentContext segmentContext = null;
    Object child = null;
    List<Object> collectionChild = null;
    if (depth == 0 && path.hasCollectionRoot()) {
        // if root is a collection
        collectionChild = (List<Object>) parent;
        segmentContext = segments.get(depth);
    } else {
        if (depth == 0) {
            // skip the root, if not a collection
            depth = 1;
        }
        segmentContext = segments.get(depth);
        String fieldName = segmentContext.getName();
        child = ((Struct) parent).get(fieldName);
        if (segmentContext.getCollectionType() != CollectionType.NONE) {
            collectionChild = (List<Object>) child;
        }
    }
    if (segmentContext.getCollectionType() == CollectionType.NONE) {
        List<Field> childFields = getFieldsForPath(session, child, field, path, depth + 1);
        fields.addAll(childFields);
        return fields;
    }
    // collection
    if (segmentContext.getCollectionIndex() != null) {
        if (collectionChild.size() <= segmentContext.getCollectionIndex()) {
            // index out of range
            return fields;
        }
        List<Field> arrayFields = getFieldsForPath(session, collectionChild.get(segmentContext.getCollectionIndex()), field, path, depth + 1);
        fields.addAll(arrayFields);
    } else {
        // if index not included, iterate over all
        for (int i = 0; i < collectionChild.size(); i++) {
            Field itemField;
            if (field instanceof FieldGroup) {
                itemField = AtlasKafkaConnectModelFactory.cloneFieldGroup((FieldGroup) field);
                AtlasPath.setCollectionIndexRecursively((FieldGroup) itemField, depth, i);
            } else {
                itemField = AtlasKafkaConnectModelFactory.cloneField((KafkaConnectField) field, false);
                AtlasPath itemPath = new AtlasPath(field.getPath());
                itemPath.setCollectionIndex(depth, i);
                itemField.setPath(itemPath.toString());
            }
            List<Field> arrayFields = getFieldsForPath(session, collectionChild.get(i), itemField, new AtlasPath(itemField.getPath()), depth + 1);
            fields.addAll(arrayFields);
        }
    }
    return fields;
}

Example 8 with KafkaConnectField

use of io.atlasmap.kafkaconnect.v2.KafkaConnectField in project atlasmap by atlasmap.

the class KafkaConnectFieldReader method populateCollectionItems.

private FieldGroup populateCollectionItems(AtlasInternalSession session, List<Object> values, Field field) throws AtlasException {
    FieldGroup group = AtlasModelFactory.createFieldGroupFrom(field, true);
    for (int i = 0; i < values.size(); i++) {
        AtlasPath itemPath = new AtlasPath(group.getPath());
        List<SegmentContext> segments = itemPath.getSegments(true);
        itemPath.setCollectionIndex(segments.size() - 1, i);
        if (field instanceof FieldGroup) {
            FieldGroup itemGroup = AtlasKafkaConnectModelFactory.cloneFieldGroup((FieldGroup) field);
            AtlasPath.setCollectionIndexRecursively(itemGroup, segments.size() - 1, i);
            populateChildFields(session, (Struct) values.get(i), itemGroup);
            group.getField().add(itemGroup);
        } else {
            KafkaConnectField itemField = AtlasKafkaConnectModelFactory.cloneField((KafkaConnectField) field, false);
            itemField.setPath(itemPath.toString());
            Object converted = conversionService.convertType(values.get(i), itemField.getFormat(), itemField.getFieldType(), null);
            itemField.setValue(converted);
            group.getField().add(itemField);
        }
    }
    return group;
}

Example 9 with KafkaConnectField

use of io.atlasmap.kafkaconnect.v2.KafkaConnectField in project atlasmap by atlasmap.

the class KafkaConnectInspector method createDocument.

private KafkaConnectDocument createDocument(org.apache.kafka.connect.data.Schema schema) {
    KafkaConnectDocument doc = AtlasKafkaConnectModelFactory.createKafkaConnectDocument();
    doc.setRootSchemaType(schema.type());
    doc.setName(schema.name());
    Schema connectSchema = schema;
    AtlasPath path;
    if (Type.ARRAY == connectSchema.type()) {
        path = new AtlasPath(AtlasPath.PATH_SEPARATOR + AtlasPath.PATH_LIST_SUFFIX);
        doc.setCollectionType(CollectionType.LIST);
        connectSchema = connectSchema.valueSchema();
    } else if (Type.MAP == connectSchema.type()) {
        path = new AtlasPath(AtlasPath.PATH_SEPARATOR + AtlasPath.PATH_MAP_SUFFIX);
        doc.setCollectionType(CollectionType.MAP);
        connectSchema = connectSchema.valueSchema();
    } else {
        path = new AtlasPath("");
    }
    doc.setPath(path.toString());
    if (connectSchema.parameters() != null) {
        doc.setEnumeration(true);
        List<KafkaConnectEnumField> symbols = doc.getEnumFields().getKafkaConnectEnumField();
        for (Entry<String, String> entry : connectSchema.parameters().entrySet()) {
            if ("io.confluent".equals(entry.getKey())) {
                continue;
            }
            KafkaConnectEnumField f = new KafkaConnectEnumField();
            f.setName(entry.getValue());
            symbols.add(f);
        }
        doc.setFieldType(KafkaConnectUtil.getFieldType(connectSchema.type()));
    } else if (!connectSchema.type().isPrimitive()) {
        doc.setFieldType(FieldType.COMPLEX);
        List<KafkaConnectField> children = populateFields(connectSchema.fields(), path);
        doc.getFields().getField().addAll(children);
    } else {
        doc.setFieldType(KafkaConnectUtil.getFieldType(connectSchema.type()));
    }
    return doc;
}

Example 10 with KafkaConnectField

use of io.atlasmap.kafkaconnect.v2.KafkaConnectField in project atlasmap by atlasmap.

the class KafkaConnectInspector method populateFields.

private List<KafkaConnectField> populateFields(List<org.apache.kafka.connect.data.Field> kcFields, AtlasPath parentPath) {
    List<KafkaConnectField> answer = new ArrayList<>();
    for (org.apache.kafka.connect.data.Field connectField : kcFields) {
        KafkaConnectField field;
        AtlasPath path = parentPath.clone();
        CollectionType collectionType = CollectionType.NONE;
        Schema connectSchema = connectField.schema();
        if (Type.ARRAY == connectSchema.type()) {
            path.appendField(connectField.name() + AtlasPath.PATH_LIST_SUFFIX);
            collectionType = CollectionType.LIST;
            connectSchema = connectSchema.valueSchema();
        } else if (Type.MAP == connectSchema.type()) {
            path.appendField(connectField.name() + AtlasPath.PATH_MAP_SUFFIX);
            collectionType = CollectionType.MAP;
            connectSchema = connectSchema.valueSchema();
        } else {
            path.appendField(connectField.name());
        }
        if (connectSchema.parameters() != null) {
            KafkaConnectComplexType complex = AtlasKafkaConnectModelFactory.createKafkaConnectComplexType();
            complex.setKafkaConnectEnumFields(new KafkaConnectEnumFields());
            complex.setEnumeration(true);
            List<KafkaConnectEnumField> symbols = complex.getKafkaConnectEnumFields().getKafkaConnectEnumField();
            boolean first = true;
            for (Entry<String, String> entry : connectSchema.parameters().entrySet()) {
                // FIXME dirty hack - it seems the first entry is for some control, the others are the actual values
                if (first) {
                    first = false;
                    continue;
                }
                KafkaConnectEnumField f = new KafkaConnectEnumField();
                f.setName(entry.getValue());
                symbols.add(f);
            }
            field = complex;
        } else if (connectSchema.type().isPrimitive()) {
            field = AtlasKafkaConnectModelFactory.createKafkaConnectField();
            field.setFieldType(KafkaConnectUtil.getFieldType(connectSchema.type()));
        } else {
            KafkaConnectComplexType complex = AtlasKafkaConnectModelFactory.createKafkaConnectComplexType();
            List<KafkaConnectField> children = populateFields(connectSchema.fields(), path);
            if ("io.confluent.connect.avro.Union".equals(connectSchema.name())) {
                // We don't support union until it's built into Kafka Connect Schema
                complex.setStatus(FieldStatus.UNSUPPORTED);
            }
            complex.getKafkaConnectFields().getKafkaConnectField().addAll(children);
            field = complex;
        }
        field.setName(connectField.name());
        field.setPath(path.toString());
        field.setCollectionType(collectionType);
        answer.add(field);
    }
    return answer;
}