Search in sources :

Example 1 with CREATED

use of org.apache.nifi.atlas.NiFiFlow.EntityChangeType.CREATED in project nifi by apache.

the class NiFiAtlasClient method registerDataSetEntities.

/**
 * Register DataSet within specified NiFiFlow.
 * @return Set of registered Atlas type names and its remaining entities without deleted ones.
 */
private Map<String, List<AtlasEntity>> registerDataSetEntities(final NiFiFlow nifiFlow) throws AtlasServiceException {
    final Map<NiFiFlow.EntityChangeType, List<AtlasEntity>> changedEntities = nifiFlow.getChangedDataSetEntities();
    if (changedEntities.containsKey(CREATED)) {
        final List<AtlasEntity> createdEntities = changedEntities.get(CREATED);
        final AtlasEntity.AtlasEntitiesWithExtInfo atlasEntities = new AtlasEntity.AtlasEntitiesWithExtInfo(createdEntities);
        final EntityMutationResponse mutationResponse = atlasClient.createEntities(atlasEntities);
        logger.debug("Created DataSet entities mutation response={}", mutationResponse);
        final Map<String, String> guidAssignments = mutationResponse.getGuidAssignments();
        for (AtlasEntity entity : createdEntities) {
            final String guid = guidAssignments.get(entity.getGuid());
            final String qualifiedName = toStr(entity.getAttribute(ATTR_QUALIFIED_NAME));
            if (StringUtils.isEmpty(guid)) {
                logger.warn("GUID was not assigned for {}::{} for some reason.", entity.getTypeName(), qualifiedName);
                continue;
            }
            final Map<AtlasObjectId, AtlasEntity> entityMap;
            switch(entity.getTypeName()) {
                case TYPE_NIFI_INPUT_PORT:
                    entityMap = nifiFlow.getRootInputPortEntities();
                    break;
                case TYPE_NIFI_OUTPUT_PORT:
                    entityMap = nifiFlow.getRootOutputPortEntities();
                    break;
                case TYPE_NIFI_QUEUE:
                    entityMap = nifiFlow.getQueues();
                    break;
                default:
                    throw new RuntimeException(entity.getTypeName() + " is not expected.");
            }
            // In order to replace the id, remove current id which does not have GUID.
            findIdByQualifiedName(entityMap.keySet(), qualifiedName).ifPresent(entityMap::remove);
            entity.setGuid(guid);
            final AtlasObjectId idWithGuid = new AtlasObjectId(guid, entity.getTypeName(), Collections.singletonMap(ATTR_QUALIFIED_NAME, qualifiedName));
            entityMap.put(idWithGuid, entity);
        }
    }
    if (changedEntities.containsKey(UPDATED)) {
        final List<AtlasEntity> updatedEntities = changedEntities.get(UPDATED);
        final AtlasEntity.AtlasEntitiesWithExtInfo atlasEntities = new AtlasEntity.AtlasEntitiesWithExtInfo(updatedEntities);
        final EntityMutationResponse mutationResponse = atlasClient.updateEntities(atlasEntities);
        logger.debug("Updated DataSet entities mutation response={}", mutationResponse);
    }
    final Set<String> changedTypeNames = changedEntities.entrySet().stream().filter(entry -> !AS_IS.equals(entry.getKey())).flatMap(entry -> entry.getValue().stream()).map(AtlasEntity::getTypeName).collect(Collectors.toSet());
    // NOTE: Cascading DELETE will be performed when parent NiFiFlow is updated without removed DataSet entities.
    final Map<String, List<AtlasEntity>> remainingEntitiesByType = changedEntities.entrySet().stream().filter(entry -> !DELETED.equals(entry.getKey())).flatMap(entry -> entry.getValue().stream()).filter(entity -> changedTypeNames.contains(entity.getTypeName())).collect(Collectors.groupingBy(AtlasEntity::getTypeName));
    // If all entities are deleted for a type (e.g. nifi_intput_port), then remainingEntitiesByType will not contain such key.
    // If the returning map does not contain anything for a type, then the corresponding attribute will not be updated.
    // To empty an attribute when all of its elements are deleted, add empty list for a type.
    changedTypeNames.forEach(changedTypeName -> remainingEntitiesByType.computeIfAbsent(changedTypeName, k -> Collections.emptyList()));
    return remainingEntitiesByType;
}
Also used : AtlasAttributeDef(org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef) ATTR_INPUT_PORTS(org.apache.nifi.atlas.NiFiTypes.ATTR_INPUT_PORTS) TYPE_NIFI_FLOW(org.apache.nifi.atlas.NiFiTypes.TYPE_NIFI_FLOW) LoggerFactory(org.slf4j.LoggerFactory) ATTR_OUTPUT_PORTS(org.apache.nifi.atlas.NiFiTypes.ATTR_OUTPUT_PORTS) ATTR_QUEUES(org.apache.nifi.atlas.NiFiTypes.ATTR_QUEUES) DELETED(org.apache.nifi.atlas.NiFiFlow.EntityChangeType.DELETED) ATTR_QUALIFIED_NAME(org.apache.nifi.atlas.NiFiTypes.ATTR_QUALIFIED_NAME) UPDATED(org.apache.nifi.atlas.NiFiFlow.EntityChangeType.UPDATED) Matcher(java.util.regex.Matcher) AtlasErrorCode(org.apache.atlas.AtlasErrorCode) AtlasObjectId(org.apache.atlas.model.instance.AtlasObjectId) AtlasTypesDef(org.apache.atlas.model.typedef.AtlasTypesDef) Map(java.util.Map) TYPE_NIFI_OUTPUT_PORT(org.apache.nifi.atlas.NiFiTypes.TYPE_NIFI_OUTPUT_PORT) CREATED(org.apache.nifi.atlas.NiFiFlow.EntityChangeType.CREATED) AtlasUtils.getComponentIdFromQualifiedName(org.apache.nifi.atlas.AtlasUtils.getComponentIdFromQualifiedName) AtlasEntityDef(org.apache.atlas.model.typedef.AtlasEntityDef) MultivaluedMapImpl(com.sun.jersey.core.util.MultivaluedMapImpl) AtlasServiceException(org.apache.atlas.AtlasServiceException) Set(java.util.Set) StringUtils(org.apache.nifi.util.StringUtils) Collectors(java.util.stream.Collectors) Objects(java.util.Objects) List(java.util.List) ATTR_URL(org.apache.nifi.atlas.NiFiTypes.ATTR_URL) Pattern(java.util.regex.Pattern) UniformInterfaceException(com.sun.jersey.api.client.UniformInterfaceException) AtlasClientV2(org.apache.atlas.AtlasClientV2) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) HashMap(java.util.HashMap) EntityMutationResponse(org.apache.atlas.model.instance.EntityMutationResponse) Function(java.util.function.Function) ArrayList(java.util.ArrayList) HashSet(java.util.HashSet) ATTR_OUTPUTS(org.apache.nifi.atlas.NiFiTypes.ATTR_OUTPUTS) TYPE_NIFI_FLOW_PATH(org.apache.nifi.atlas.NiFiTypes.TYPE_NIFI_FLOW_PATH) AtlasUtils.findIdByQualifiedName(org.apache.nifi.atlas.AtlasUtils.findIdByQualifiedName) AS_IS(org.apache.nifi.atlas.NiFiFlow.EntityChangeType.AS_IS) AtlasEntity(org.apache.atlas.model.instance.AtlasEntity) ATTR_INPUTS(org.apache.nifi.atlas.NiFiTypes.ATTR_INPUTS) ATTR_NAME(org.apache.nifi.atlas.NiFiTypes.ATTR_NAME) Logger(org.slf4j.Logger) AtlasUtils.toStr(org.apache.nifi.atlas.AtlasUtils.toStr) ATTR_FLOW_PATHS(org.apache.nifi.atlas.NiFiTypes.ATTR_FLOW_PATHS) TYPE_NIFI_INPUT_PORT(org.apache.nifi.atlas.NiFiTypes.TYPE_NIFI_INPUT_PORT) ATTR_DESCRIPTION(org.apache.nifi.atlas.NiFiTypes.ATTR_DESCRIPTION) ENTITIES(org.apache.nifi.atlas.NiFiTypes.ENTITIES) ATTR_GUID(org.apache.nifi.atlas.NiFiTypes.ATTR_GUID) MultivaluedMap(javax.ws.rs.core.MultivaluedMap) Tuple(org.apache.nifi.util.Tuple) TYPE_NIFI_QUEUE(org.apache.nifi.atlas.NiFiTypes.TYPE_NIFI_QUEUE) SearchFilter(org.apache.atlas.model.SearchFilter) ATTR_TYPENAME(org.apache.nifi.atlas.NiFiTypes.ATTR_TYPENAME) Collections(java.util.Collections) EntityMutationResponse(org.apache.atlas.model.instance.EntityMutationResponse) AtlasObjectId(org.apache.atlas.model.instance.AtlasObjectId) AtlasEntity(org.apache.atlas.model.instance.AtlasEntity) List(java.util.List) ArrayList(java.util.ArrayList)

Example 2 with CREATED

use of org.apache.nifi.atlas.NiFiFlow.EntityChangeType.CREATED in project nifi by apache.

the class NiFiAtlasClient method registerFlowPathEntities.

private Set<AtlasObjectId> registerFlowPathEntities(final NiFiFlow nifiFlow) throws AtlasServiceException {
    final Map<NiFiFlow.EntityChangeType, List<AtlasEntity>> changedEntities = nifiFlow.getChangedFlowPathEntities();
    if (changedEntities.containsKey(CREATED)) {
        final List<AtlasEntity> createdEntities = changedEntities.get(CREATED);
        final AtlasEntity.AtlasEntitiesWithExtInfo atlasEntities = new AtlasEntity.AtlasEntitiesWithExtInfo(createdEntities);
        final EntityMutationResponse mutationResponse = atlasClient.createEntities(atlasEntities);
        logger.debug("Created FlowPath entities mutation response={}", mutationResponse);
        final Map<String, String> guidAssignments = mutationResponse.getGuidAssignments();
        createdEntities.forEach(entity -> {
            final String guid = entity.getGuid();
            entity.setGuid(guidAssignments.get(guid));
            final String pathId = getComponentIdFromQualifiedName(toStr(entity.getAttribute(ATTR_QUALIFIED_NAME)));
            final NiFiFlowPath path = nifiFlow.getFlowPaths().get(pathId);
            path.setExEntity(entity);
        });
    }
    if (changedEntities.containsKey(UPDATED)) {
        final List<AtlasEntity> updatedEntities = changedEntities.get(UPDATED);
        final AtlasEntity.AtlasEntitiesWithExtInfo atlasEntities = new AtlasEntity.AtlasEntitiesWithExtInfo(updatedEntities);
        final EntityMutationResponse mutationResponse = atlasClient.updateEntities(atlasEntities);
        logger.debug("Updated FlowPath entities mutation response={}", mutationResponse);
        updatedEntities.forEach(entity -> {
            final String pathId = getComponentIdFromQualifiedName(toStr(entity.getAttribute(ATTR_QUALIFIED_NAME)));
            final NiFiFlowPath path = nifiFlow.getFlowPaths().get(pathId);
            path.setExEntity(entity);
        });
    }
    if (NiFiFlow.EntityChangeType.containsChange(changedEntities.keySet())) {
        return changedEntities.entrySet().stream().filter(entry -> !DELETED.equals(entry.getKey())).flatMap(entry -> entry.getValue().stream()).map(path -> new AtlasObjectId(path.getGuid(), TYPE_NIFI_FLOW_PATH, Collections.singletonMap(ATTR_QUALIFIED_NAME, path.getAttribute(ATTR_QUALIFIED_NAME)))).collect(Collectors.toSet());
    }
    return null;
}
Also used : AtlasAttributeDef(org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef) ATTR_INPUT_PORTS(org.apache.nifi.atlas.NiFiTypes.ATTR_INPUT_PORTS) TYPE_NIFI_FLOW(org.apache.nifi.atlas.NiFiTypes.TYPE_NIFI_FLOW) LoggerFactory(org.slf4j.LoggerFactory) ATTR_OUTPUT_PORTS(org.apache.nifi.atlas.NiFiTypes.ATTR_OUTPUT_PORTS) ATTR_QUEUES(org.apache.nifi.atlas.NiFiTypes.ATTR_QUEUES) DELETED(org.apache.nifi.atlas.NiFiFlow.EntityChangeType.DELETED) ATTR_QUALIFIED_NAME(org.apache.nifi.atlas.NiFiTypes.ATTR_QUALIFIED_NAME) UPDATED(org.apache.nifi.atlas.NiFiFlow.EntityChangeType.UPDATED) Matcher(java.util.regex.Matcher) AtlasErrorCode(org.apache.atlas.AtlasErrorCode) AtlasObjectId(org.apache.atlas.model.instance.AtlasObjectId) AtlasTypesDef(org.apache.atlas.model.typedef.AtlasTypesDef) Map(java.util.Map) TYPE_NIFI_OUTPUT_PORT(org.apache.nifi.atlas.NiFiTypes.TYPE_NIFI_OUTPUT_PORT) CREATED(org.apache.nifi.atlas.NiFiFlow.EntityChangeType.CREATED) AtlasUtils.getComponentIdFromQualifiedName(org.apache.nifi.atlas.AtlasUtils.getComponentIdFromQualifiedName) AtlasEntityDef(org.apache.atlas.model.typedef.AtlasEntityDef) MultivaluedMapImpl(com.sun.jersey.core.util.MultivaluedMapImpl) AtlasServiceException(org.apache.atlas.AtlasServiceException) Set(java.util.Set) StringUtils(org.apache.nifi.util.StringUtils) Collectors(java.util.stream.Collectors) Objects(java.util.Objects) List(java.util.List) ATTR_URL(org.apache.nifi.atlas.NiFiTypes.ATTR_URL) Pattern(java.util.regex.Pattern) UniformInterfaceException(com.sun.jersey.api.client.UniformInterfaceException) AtlasClientV2(org.apache.atlas.AtlasClientV2) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) HashMap(java.util.HashMap) EntityMutationResponse(org.apache.atlas.model.instance.EntityMutationResponse) Function(java.util.function.Function) ArrayList(java.util.ArrayList) HashSet(java.util.HashSet) ATTR_OUTPUTS(org.apache.nifi.atlas.NiFiTypes.ATTR_OUTPUTS) TYPE_NIFI_FLOW_PATH(org.apache.nifi.atlas.NiFiTypes.TYPE_NIFI_FLOW_PATH) AtlasUtils.findIdByQualifiedName(org.apache.nifi.atlas.AtlasUtils.findIdByQualifiedName) AS_IS(org.apache.nifi.atlas.NiFiFlow.EntityChangeType.AS_IS) AtlasEntity(org.apache.atlas.model.instance.AtlasEntity) ATTR_INPUTS(org.apache.nifi.atlas.NiFiTypes.ATTR_INPUTS) ATTR_NAME(org.apache.nifi.atlas.NiFiTypes.ATTR_NAME) Logger(org.slf4j.Logger) AtlasUtils.toStr(org.apache.nifi.atlas.AtlasUtils.toStr) ATTR_FLOW_PATHS(org.apache.nifi.atlas.NiFiTypes.ATTR_FLOW_PATHS) TYPE_NIFI_INPUT_PORT(org.apache.nifi.atlas.NiFiTypes.TYPE_NIFI_INPUT_PORT) ATTR_DESCRIPTION(org.apache.nifi.atlas.NiFiTypes.ATTR_DESCRIPTION) ENTITIES(org.apache.nifi.atlas.NiFiTypes.ENTITIES) ATTR_GUID(org.apache.nifi.atlas.NiFiTypes.ATTR_GUID) MultivaluedMap(javax.ws.rs.core.MultivaluedMap) Tuple(org.apache.nifi.util.Tuple) TYPE_NIFI_QUEUE(org.apache.nifi.atlas.NiFiTypes.TYPE_NIFI_QUEUE) SearchFilter(org.apache.atlas.model.SearchFilter) ATTR_TYPENAME(org.apache.nifi.atlas.NiFiTypes.ATTR_TYPENAME) Collections(java.util.Collections) EntityMutationResponse(org.apache.atlas.model.instance.EntityMutationResponse) AtlasObjectId(org.apache.atlas.model.instance.AtlasObjectId) AtlasEntity(org.apache.atlas.model.instance.AtlasEntity) List(java.util.List) ArrayList(java.util.ArrayList)

Aggregations

UniformInterfaceException (com.sun.jersey.api.client.UniformInterfaceException)2 MultivaluedMapImpl (com.sun.jersey.core.util.MultivaluedMapImpl)2 ArrayList (java.util.ArrayList)2 Collections (java.util.Collections)2 HashMap (java.util.HashMap)2 HashSet (java.util.HashSet)2 List (java.util.List)2 Map (java.util.Map)2 Objects (java.util.Objects)2 Set (java.util.Set)2 AtomicBoolean (java.util.concurrent.atomic.AtomicBoolean)2 Function (java.util.function.Function)2 Matcher (java.util.regex.Matcher)2 Pattern (java.util.regex.Pattern)2 Collectors (java.util.stream.Collectors)2 MultivaluedMap (javax.ws.rs.core.MultivaluedMap)2 AtlasClientV2 (org.apache.atlas.AtlasClientV2)2 AtlasErrorCode (org.apache.atlas.AtlasErrorCode)2 AtlasServiceException (org.apache.atlas.AtlasServiceException)2 SearchFilter (org.apache.atlas.model.SearchFilter)2