Examples with Collection io.atlasmap.v2.Collection used on opensource projects

Example 31 with Collection

use of io.atlasmap.v2.Collection in project atlasmap by atlasmap.

the class DefaultAtlasPreviewContextTest method testProcessPreviewSplitCollection.

@Test
public void testProcessPreviewSplitCollection() throws AtlasException {
    Mapping m = new Mapping();
    Field source = new SimpleField();
    source.setFieldType(FieldType.STRING);
    source.setPath("/source");
    source.setValue("one two three four");
    source.setActions(new ArrayList<>());
    Split action = new Split();
    action.setDelimiter(" ");
    source.getActions().add(action);
    m.getInputField().add(source);
    Field target = new SimpleField();
    target.setCollectionType(CollectionType.LIST);
    target.setFieldType(FieldType.STRING);
    target.setPath("/results<>");
    m.getOutputField().add(target);
    Audits audits = previewContext.processPreview(m);
    assertEquals(0, audits.getAudit().size(), printAudit(audits));
    target = m.getOutputField().get(0);
    assertEquals(FieldGroup.class, target.getClass());
    FieldGroup targetGroup = (FieldGroup) target;
    Field one = targetGroup.getField().get(0);
    Field two = targetGroup.getField().get(1);
    Field three = targetGroup.getField().get(2);
    Field four = targetGroup.getField().get(3);
    assertEquals("/results<0>", one.getPath());
    assertEquals("one", one.getValue());
    assertEquals("/results<1>", two.getPath());
    assertEquals("two", two.getValue());
    assertEquals("/results<2>", three.getPath());
    assertEquals("three", three.getValue());
    assertEquals("/results<3>", four.getPath());
    assertEquals("four", four.getValue());
    target = new SimpleField();
    target.setCollectionType(CollectionType.NONE);
    target.setPath("/collection<>/result");
    target.setFieldType(FieldType.STRING);
    target.setIndex(0);
    m.getOutputField().clear();
    m.getOutputField().add(target);
    audits = previewContext.processPreview(m);
    assertEquals(0, audits.getAudit().size(), printAudit(audits));
    target = m.getOutputField().get(0);
    assertEquals(FieldGroup.class, target.getClass());
    targetGroup = (FieldGroup) target;
    one = targetGroup.getField().get(0);
    two = targetGroup.getField().get(1);
    three = targetGroup.getField().get(2);
    four = targetGroup.getField().get(3);
    assertEquals("/collection<0>/result", one.getPath());
    assertEquals("one", one.getValue());
    assertEquals("/collection<1>/result", two.getPath());
    assertEquals("two", two.getValue());
    assertEquals("/collection<2>/result", three.getPath());
    assertEquals("three", three.getValue());
    assertEquals("/collection<3>/result", four.getPath());
    assertEquals("four", four.getValue());
}

Example 32 with Collection

use of io.atlasmap.v2.Collection in project atlasmap by atlasmap.

the class XmlFieldReader method getFieldsForPath.

private List<Field> getFieldsForPath(AtlasInternalSession session, Optional<XmlNamespaces> xmlNamespaces, Element node, Field field, XmlPath path, int depth) throws AtlasException {
    List<Field> fields = new ArrayList<>();
    List<XmlSegmentContext> segments = path.getXmlSegments(false);
    if (segments.size() < depth) {
        throw new AtlasException(String.format("depth '%s' exceeds segment size '%s'", depth, segments.size()));
    }
    if (segments.size() == depth) {
        if (!(field instanceof XmlEnumField) && field.getFieldType() == FieldType.COMPLEX) {
            FieldGroup group = (FieldGroup) field;
            populateChildFields(session, xmlNamespaces, node, group, path);
            fields.add(group);
        } else {
            XmlField xmlField = new XmlField();
            AtlasXmlModelFactory.copyField(field, xmlField, true);
            if (field instanceof XmlEnumField && xmlField.getFieldType() == FieldType.COMPLEX) {
                // enum has COMPLEX by default
                xmlField.setFieldType(FieldType.STRING);
            }
            copyValue(session, xmlNamespaces, segments.get(depth - 1), node, xmlField);
            // reset index for subfields
            xmlField.setIndex(null);
            fields.add(xmlField);
        }
        return fields;
    }
    // segments.size() > depth
    XmlSegmentContext segment = segments.get(depth);
    if (LOG.isDebugEnabled()) {
        LOG.debug("Now processing segment: " + segment.getName());
    }
    if (depth == 0) {
        if (segment.getName().startsWith(XmlIOHelper.getNodeNameWithoutNamespaceAlias(node))) {
            Optional<String> rootNamespace = Optional.empty();
            if (segment.getNamespace() != null) {
                rootNamespace = getNamespace(xmlNamespaces, segment.getNamespace());
            }
            if (!rootNamespace.isPresent() || rootNamespace.get().equals(node.getNamespaceURI())) {
                // "/XOA/contact<>/firstName", skip.
                if (LOG.isDebugEnabled()) {
                    LOG.debug("Skipping root segment: " + segment);
                }
                if (segments.size() > 1) {
                    depth = 1;
                    segment = segments.get(depth);
                }
            }
        }
    }
    if (segment.isAttribute() && segments.size() == depth + 1) {
        // if last segment is attribute
        List<Field> attrFields = getFieldsForPath(session, xmlNamespaces, node, field, path, depth + 1);
        fields.addAll(attrFields);
        return fields;
    }
    String fieldName = segment.getName();
    String fieldNamespace = segment.getNamespace();
    Optional<String> namespace = getNamespace(xmlNamespaces, fieldNamespace);
    List<Element> children = XmlIOHelper.getChildrenWithNameStripAlias(fieldName, namespace, node);
    if (children == null || children.isEmpty()) {
        if (LOG.isDebugEnabled()) {
            LOG.debug("Skipping source value set, couldn't find children with name '" + fieldName + "', for segment: " + segment);
        }
        return fields;
    }
    if (segment.getCollectionType() == CollectionType.NONE) {
        List<Field> childFields = getFieldsForPath(session, xmlNamespaces, children.get(0), field, path, depth + 1);
        fields.addAll(childFields);
        return fields;
    }
    // collection
    Integer index = segment.getCollectionIndex();
    if (index != null) {
        if (index < children.size()) {
            List<Field> arrayFields = getFieldsForPath(session, xmlNamespaces, children.get(index), field, path, depth + 1);
            fields.addAll(arrayFields);
        } else if (LOG.isDebugEnabled()) {
            LOG.debug("Skipping source value set, children list can't fit index " + index + ", children list size: " + children.size());
        }
    } else {
        // if index not included, iterate over all
        for (int i = 0; i < children.size(); i++) {
            Field itemField;
            if (field instanceof FieldGroup) {
                itemField = AtlasXmlModelFactory.cloneFieldGroup((FieldGroup) field);
                AtlasPath.setCollectionIndexRecursively((FieldGroup) itemField, depth + 1, i);
            } else {
                itemField = AtlasXmlModelFactory.cloneField((XmlField) field, false);
                AtlasPath itemPath = new AtlasPath(field.getPath());
                itemPath.setCollectionIndex(depth + 1, i);
                itemField.setPath(itemPath.toString());
            }
            List<Field> arrayFields = getFieldsForPath(session, xmlNamespaces, children.get(i), itemField, new XmlPath(itemField.getPath()), depth + 1);
            fields.addAll(arrayFields);
        }
    }
    return fields;
}

Example 33 with Collection

use of io.atlasmap.v2.Collection in project atlasmap by atlasmap.

the class XmlSchemaInspector method printAttributes.

private void printAttributes(XSComplexType xsComplexType, XmlComplexType parentXmlComplexType) {
    Collection<? extends XSAttributeUse> c = xsComplexType.getDeclaredAttributeUses();
    for (XSAttributeUse aC : c) {
        XmlField xmlField = AtlasXmlModelFactory.createXmlField();
        XSAttributeDecl attributeDecl = aC.getDecl();
        xmlField.setName(getNameNS(attributeDecl));
        xmlField.setAttribute(true);
        if (attributeDecl.getDefaultValue() != null) {
            xmlField.setValue(attributeDecl.getDefaultValue().value);
        } else if (attributeDecl.getFixedValue() != null) {
            xmlField.setValue(attributeDecl.getFixedValue().value);
        }
        xmlField.setPath(parentXmlComplexType.getPath() + "/" + "@" + getNameNS(attributeDecl));
        FieldType attrType = getFieldType(attributeDecl.getType().getName());
        xmlField.setFieldType(attrType);
        if (xmlField.getFieldType() == null) {
            // check the simple types in the schema....
            XSSimpleType simpleType = xsComplexType.getRoot().getSimpleType(xsComplexType.getTargetNamespace(), attributeDecl.getType().getName());
            if (simpleType != null) {
                FieldType fieldType = getFieldType(simpleType.getBaseType().getName());
                xmlField.setFieldType(fieldType);
                xmlField.setTypeName(attributeDecl.getType().getName());
                if (simpleType.asRestriction() != null) {
                    mapRestrictions(xmlField, simpleType.asRestriction());
                }
            } else {
                // cannot figure it out....
                xmlField.setFieldType(FieldType.UNSUPPORTED);
            }
        }
        parentXmlComplexType.getXmlFields().getXmlField().add(xmlField);
    }
}

Example 34 with Collection

use of io.atlasmap.v2.Collection in project atlasmap by atlasmap.

the class KafkaConnectFieldReader method nestComplexCollection.

private void nestComplexCollection(AtlasInternalSession session, List<Object> collection, FieldGroup parent, int depth) throws AtlasException {
    AtlasPath path = new AtlasPath(parent.getPath());
    SegmentContext segment = path.getSegments(true).get(depth);
    if (segment.getCollectionIndex() != null) {
        int index = segment.getCollectionIndex();
        Struct struct = (Struct) collection.get(index);
        populateChildFields(session, struct, parent);
        return;
    }
    List<Field> processed = new LinkedList<>();
    for (int index = 0; index < collection.size(); index++) {
        FieldGroup itemGroup = AtlasKafkaConnectModelFactory.cloneFieldGroup(parent);
        AtlasPath.setCollectionIndexRecursively(itemGroup, depth, index);
        processed.add(itemGroup);
        Struct struct = (Struct) collection.get(index);
        populateChildFields(session, struct, itemGroup);
    }
    parent.getField().clear();
    parent.getField().addAll(processed);
}

Example 35 with Collection

use of io.atlasmap.v2.Collection in project atlasmap by atlasmap.

the class DefaultAtlasContext method processTargetFieldMapping.

private void processTargetFieldMapping(DefaultAtlasSession session, Mapping mapping) {
    MappingType mappingType = mapping.getMappingType();
    List<Field> sourceFields = mapping.getInputField();
    List<Field> targetFields = mapping.getOutputField();
    AtlasModule module = null;
    Field targetField = null;
    if (mappingType == null || mappingType == MappingType.LOOKUP || mappingType == MappingType.MAP) {
        Field sourceField = session.head().getSourceField();
        FieldGroup sourceFieldGroup = null;
        if (sourceField instanceof FieldGroup) {
            sourceFieldGroup = unwrapNestedGroup((FieldGroup) sourceField);
        }
        for (Field f : targetFields) {
            targetField = f;
            module = resolveModule(FieldDirection.TARGET, targetField);
            if (!auditTargetFieldType(session, module, targetField)) {
                continue;
            }
            session.head().setTargetField(targetField);
            if (sourceFieldGroup != null) {
                Integer index = targetField.getIndex();
                AtlasPath targetPath = new AtlasPath(targetField.getPath());
                if (targetPath.hasCollection() && !targetPath.isIndexedCollection()) {
                    if (targetFields.size() > 1) {
                        AtlasUtil.addAudit(session, targetField, "It's not yet supported to have a collection field as a part of multiple target fields in a same mapping", AuditStatus.ERROR, null);
                        return;
                    }
                    session.head().setSourceField(sourceFieldGroup);
                } else if (index == null) {
                    if (sourceFieldGroup.getField().size() > 0) {
                        session.head().setSourceField(sourceFieldGroup.getField().get(sourceFieldGroup.getField().size() - 1));
                    }
                } else {
                    if (sourceFieldGroup.getField().size() > index) {
                        session.head().setSourceField(sourceFieldGroup.getField().get(index));
                    } else {
                        AtlasUtil.addAudit(session, targetField, String.format("The number of source fields '%s' is fewer than expected via target field index '%s'", sourceFieldGroup.getField().size(), targetField.getIndex()), AuditStatus.WARN, null);
                        continue;
                    }
                }
            }
            try {
                module.populateTargetField(session);
            } catch (Exception e) {
                AtlasUtil.addAudit(session, targetField, "Failed to populate target field: " + e.getMessage(), AuditStatus.ERROR, null);
                if (LOG.isDebugEnabled()) {
                    LOG.error(String.format("populateTargetField() failed for %s:%s", targetField.getDocId(), targetField.getPath()), e);
                }
                return;
            }
            Field processed = applyFieldActions(session, session.head().getTargetField());
            session.head().setTargetField(processed);
            try {
                module.writeTargetValue(session);
            } catch (Exception e) {
                AtlasUtil.addAudit(session, targetField, "Failed to write field value into target document: " + e.getMessage(), AuditStatus.ERROR, null);
                if (LOG.isDebugEnabled()) {
                    LOG.error(String.format("writeTargetValue() failed for %s:%s", targetField.getDocId(), targetField.getPath()), e);
                }
                return;
            }
        }
        return;
    } else if (mappingType == MappingType.COMBINE) {
        targetField = targetFields.get(0);
        module = resolveModule(FieldDirection.TARGET, targetField);
        if (!auditTargetFieldType(session, module, targetField)) {
            return;
        }
        Field sourceField = processCombineField(session, mapping, sourceFields, targetField);
        session.head().setSourceField(sourceField).setTargetField(targetField);
        try {
            module.populateTargetField(session);
        } catch (Exception e) {
            AtlasUtil.addAudit(session, targetField, "Failed to populate target field: " + e.getMessage(), AuditStatus.ERROR, null);
            return;
        }
        applyFieldActions(session, session.head().getTargetField());
        try {
            module.writeTargetValue(session);
        } catch (Exception e) {
            AtlasUtil.addAudit(session, targetField, "Failed to write field value into target document: " + e.getMessage(), AuditStatus.ERROR, null);
            return;
        }
        return;
    } else if (mappingType == MappingType.SEPARATE) {
        List<Field> separatedFields = null;
        try {
            separatedFields = processSeparateField(session, mapping, sourceFields.get(0));
        } catch (Exception e) {
            AtlasUtil.addAudit(session, targetField, "Failed to process separate mode: " + e.getMessage(), AuditStatus.ERROR, null);
            return;
        }
        if (separatedFields == null) {
            return;
        }
        for (Field f : targetFields) {
            targetField = f;
            module = resolveModule(FieldDirection.TARGET, targetField);
            if (!auditTargetFieldType(session, module, targetField)) {
                continue;
            }
            if (targetField.getIndex() == null || targetField.getIndex() < 0) {
                AtlasUtil.addAudit(session, targetField, String.format("Separate requires zero or positive Index value to be set on targetField targetField.path=%s", targetField.getPath()), AuditStatus.WARN, null);
                continue;
            }
            if (separatedFields.size() <= targetField.getIndex()) {
                String errorMessage = String.format("Separate returned fewer segments count=%s when targetField.path=%s requested index=%s", separatedFields.size(), targetField.getPath(), targetField.getIndex());
                AtlasUtil.addAudit(session, targetField, errorMessage, AuditStatus.WARN, null);
                break;
            }
            session.head().setSourceField(separatedFields.get(targetField.getIndex())).setTargetField(targetField);
            try {
                module.populateTargetField(session);
            } catch (Exception e) {
                AtlasUtil.addAudit(session, targetField, "Failed to populate target field: " + e.getMessage(), AuditStatus.ERROR, null);
                return;
            }
            Field processed = applyFieldActions(session, session.head().getTargetField());
            session.head().setTargetField(processed);
            try {
                module.writeTargetValue(session);
            } catch (Exception e) {
                AtlasUtil.addAudit(session, targetField, "Failed to write field value into target document: " + e.getMessage(), AuditStatus.ERROR, null);
                return;
            }
        }
        return;
    }
    AtlasUtil.addAudit(session, (String) null, String.format("Unsupported mappingType=%s detected", mapping.getMappingType()), AuditStatus.ERROR, null);
}