Examples with KafkaConnectComplexType io.atlasmap.kafkaconnect.v2.KafkaConnectComplexType used on opensource projects

Example 1 with KafkaConnectComplexType

use of io.atlasmap.kafkaconnect.v2.KafkaConnectComplexType in project atlasmap by atlasmap.

the class KafkaConnectInspectionServiceTest method testAvroComplex.

@Test
public void testAvroComplex() throws Exception {
    InputStream is = Thread.currentThread().getContextClassLoader().getResourceAsStream("avro-complex.json");
    KafkaConnectDocument doc = service.inspectAvro(new String(is.readAllBytes()), options);
    assertNotNull(doc);
    assertEquals("root", doc.getName());
    assertEquals("/", doc.getPath());
    assertEquals(FieldType.COMPLEX, doc.getFieldType());
    List<Field> fields = doc.getFields().getField();
    assertEquals(9, fields.size());
    Field f1 = fields.get(0);
    assertEquals("f1", f1.getName());
    assertEquals("/f1", f1.getPath());
    assertEquals(FieldType.BOOLEAN, f1.getFieldType());
    Field f2 = fields.get(1);
    assertEquals("f2", f2.getName());
    assertEquals("/f2", f2.getPath());
    assertEquals(FieldType.STRING, f2.getFieldType());
    Field record = fields.get(2);
    assertEquals("record", record.getName());
    assertEquals("/record", record.getPath());
    assertEquals(FieldType.COMPLEX, record.getFieldType());
    assertTrue(record instanceof KafkaConnectComplexType);
    KafkaConnectComplexType recordComplex = (KafkaConnectComplexType) record;
    assertEquals(2, recordComplex.getKafkaConnectFields().getKafkaConnectField().size());
    Field recordf1 = recordComplex.getKafkaConnectFields().getKafkaConnectField().get(0);
    assertEquals("recordf1", recordf1.getName());
    assertEquals("/record/recordf1", recordf1.getPath());
    assertEquals(FieldType.LONG, recordf1.getFieldType());
    Field recordf2 = recordComplex.getKafkaConnectFields().getKafkaConnectField().get(1);
    assertEquals("recordf2", recordf2.getName());
    assertEquals("/record/recordf2", recordf2.getPath());
    assertEquals(FieldType.DOUBLE, recordf2.getFieldType());
    Field enumf = fields.get(3);
    assertEquals("enum", enumf.getName());
    assertEquals("/enum", enumf.getPath());
    assertEquals(FieldType.COMPLEX, enumf.getFieldType());
    assertTrue(record instanceof KafkaConnectComplexType);
    KafkaConnectComplexType enumfComplex = (KafkaConnectComplexType) enumf;
    List<KafkaConnectEnumField> entries = enumfComplex.getKafkaConnectEnumFields().getKafkaConnectEnumField();
    assertEquals(3, entries.size());
    assertEquals("ONE", entries.get(0).getName());
    assertEquals("TWO", entries.get(1).getName());
    assertEquals("THREE", entries.get(2).getName());
    Field sarray = fields.get(4);
    assertEquals("sarray", sarray.getName());
    assertEquals("/sarray<>", sarray.getPath());
    assertEquals(CollectionType.LIST, sarray.getCollectionType());
    assertEquals(FieldType.STRING, sarray.getFieldType());
    Field rarray = fields.get(5);
    assertEquals("rarray", rarray.getName());
    assertEquals("/rarray<>", rarray.getPath());
    assertEquals(CollectionType.LIST, rarray.getCollectionType());
    assertEquals(FieldType.COMPLEX, rarray.getFieldType());
    assertTrue(rarray instanceof KafkaConnectComplexType);
    KafkaConnectComplexType rarrayComplex = (KafkaConnectComplexType) rarray;
    List<KafkaConnectField> rarrayEntries = rarrayComplex.getKafkaConnectFields().getKafkaConnectField();
    assertEquals(2, rarrayEntries.size());
    Field rarrayf3 = rarrayEntries.get(0);
    assertEquals("recordf3", rarrayf3.getName());
    assertEquals("/rarray<>/recordf3", rarrayf3.getPath());
    assertEquals(FieldType.INTEGER, rarrayf3.getFieldType());
    Field rarrayf4 = rarrayEntries.get(1);
    assertEquals("recordf4", rarrayf4.getName());
    assertEquals("/rarray<>/recordf4", rarrayf4.getPath());
    assertEquals(FieldType.FLOAT, rarrayf4.getFieldType());
    Field map = fields.get(6);
    assertEquals("map", map.getName());
    assertEquals("/map{}", map.getPath());
    assertEquals(CollectionType.MAP, map.getCollectionType());
    assertEquals(FieldType.STRING, map.getFieldType());
    Field union = fields.get(7);
    assertEquals("union", union.getName());
    assertEquals("/union", union.getPath());
    assertEquals(FieldType.COMPLEX, union.getFieldType());
    assertTrue(union instanceof KafkaConnectComplexType);
    KafkaConnectComplexType unionComplex = (KafkaConnectComplexType) union;
    assertEquals(FieldStatus.UNSUPPORTED, unionComplex.getStatus());
    Field fixed = fields.get(8);
    assertEquals("fixed", fixed.getName());
    assertEquals("/fixed", fixed.getPath());
    assertEquals(FieldType.BYTE_ARRAY, fixed.getFieldType());
}

Example 2 with KafkaConnectComplexType

use of io.atlasmap.kafkaconnect.v2.KafkaConnectComplexType in project atlasmap by atlasmap.

the class KafkaConnectInspector method populateFields.

private List<KafkaConnectField> populateFields(List<org.apache.kafka.connect.data.Field> kcFields, AtlasPath parentPath) {
    List<KafkaConnectField> answer = new ArrayList<>();
    for (org.apache.kafka.connect.data.Field connectField : kcFields) {
        KafkaConnectField field;
        AtlasPath path = parentPath.clone();
        CollectionType collectionType = CollectionType.NONE;
        Schema connectSchema = connectField.schema();
        if (Type.ARRAY == connectSchema.type()) {
            path.appendField(connectField.name() + AtlasPath.PATH_LIST_SUFFIX);
            collectionType = CollectionType.LIST;
            connectSchema = connectSchema.valueSchema();
        } else if (Type.MAP == connectSchema.type()) {
            path.appendField(connectField.name() + AtlasPath.PATH_MAP_SUFFIX);
            collectionType = CollectionType.MAP;
            connectSchema = connectSchema.valueSchema();
        } else {
            path.appendField(connectField.name());
        }
        if (connectSchema.parameters() != null) {
            KafkaConnectComplexType complex = AtlasKafkaConnectModelFactory.createKafkaConnectComplexType();
            complex.setKafkaConnectEnumFields(new KafkaConnectEnumFields());
            complex.setEnumeration(true);
            List<KafkaConnectEnumField> symbols = complex.getKafkaConnectEnumFields().getKafkaConnectEnumField();
            boolean first = true;
            for (Entry<String, String> entry : connectSchema.parameters().entrySet()) {
                // FIXME dirty hack - it seems the first entry is for some control, the others are the actual values
                if (first) {
                    first = false;
                    continue;
                }
                KafkaConnectEnumField f = new KafkaConnectEnumField();
                f.setName(entry.getValue());
                symbols.add(f);
            }
            field = complex;
        } else if (connectSchema.type().isPrimitive()) {
            field = AtlasKafkaConnectModelFactory.createKafkaConnectField();
            field.setFieldType(KafkaConnectUtil.getFieldType(connectSchema.type()));
        } else {
            KafkaConnectComplexType complex = AtlasKafkaConnectModelFactory.createKafkaConnectComplexType();
            List<KafkaConnectField> children = populateFields(connectSchema.fields(), path);
            if ("io.confluent.connect.avro.Union".equals(connectSchema.name())) {
                // We don't support union until it's built into Kafka Connect Schema
                complex.setStatus(FieldStatus.UNSUPPORTED);
            }
            complex.getKafkaConnectFields().getKafkaConnectField().addAll(children);
            field = complex;
        }
        field.setName(connectField.name());
        field.setPath(path.toString());
        field.setCollectionType(collectionType);
        answer.add(field);
    }
    return answer;
}