Examples with AtlasObjectId org.apache.atlas.model.instance.AtlasObjectId used on opensource projects

Example 1 with AtlasObjectId

use of org.apache.atlas.model.instance.AtlasObjectId in project nifi by apache.

the class NiFiAtlasClient method fetchNiFiFlow.

/**
 * Fetch existing NiFiFlow entity from Atlas.
 * @param rootProcessGroupId The id of a NiFi flow root process group.
 * @param clusterName The cluster name of a flow.
 * @return A NiFiFlow instance filled with retrieved data from Atlas. Status objects are left blank, e.g. ProcessorStatus.
 * @throws AtlasServiceException Thrown if requesting to Atlas API failed, including when the flow is not found.
 */
public NiFiFlow fetchNiFiFlow(String rootProcessGroupId, String clusterName) throws AtlasServiceException {
    final String qualifiedName = AtlasUtils.toQualifiedName(clusterName, rootProcessGroupId);
    final AtlasObjectId flowId = new AtlasObjectId(TYPE_NIFI_FLOW, ATTR_QUALIFIED_NAME, qualifiedName);
    final AtlasEntity.AtlasEntityWithExtInfo nifiFlowExt = searchEntityDef(flowId);
    if (nifiFlowExt == null || nifiFlowExt.getEntity() == null) {
        return null;
    }
    final AtlasEntity nifiFlowEntity = nifiFlowExt.getEntity();
    final Map<String, Object> attributes = nifiFlowEntity.getAttributes();
    final NiFiFlow nifiFlow = new NiFiFlow(rootProcessGroupId);
    nifiFlow.setExEntity(nifiFlowEntity);
    nifiFlow.setFlowName(toStr(attributes.get(ATTR_NAME)));
    nifiFlow.setClusterName(clusterName);
    nifiFlow.setUrl(toStr(attributes.get(ATTR_URL)));
    nifiFlow.setDescription(toStr(attributes.get(ATTR_DESCRIPTION)));
    nifiFlow.getQueues().putAll(toQualifiedNameIds(toAtlasObjectIds(nifiFlowEntity.getAttribute(ATTR_QUEUES))));
    nifiFlow.getRootInputPortEntities().putAll(toQualifiedNameIds(toAtlasObjectIds(nifiFlowEntity.getAttribute(ATTR_INPUT_PORTS))));
    nifiFlow.getRootOutputPortEntities().putAll(toQualifiedNameIds(toAtlasObjectIds(nifiFlowEntity.getAttribute(ATTR_OUTPUT_PORTS))));
    final Map<String, NiFiFlowPath> flowPaths = nifiFlow.getFlowPaths();
    final Map<AtlasObjectId, AtlasEntity> flowPathEntities = toQualifiedNameIds(toAtlasObjectIds(attributes.get(ATTR_FLOW_PATHS)));
    for (AtlasEntity flowPathEntity : flowPathEntities.values()) {
        final String pathQualifiedName = toStr(flowPathEntity.getAttribute(ATTR_QUALIFIED_NAME));
        final NiFiFlowPath flowPath = new NiFiFlowPath(getComponentIdFromQualifiedName(pathQualifiedName));
        if (flowPathEntity.hasAttribute(ATTR_URL)) {
            final Matcher urlMatcher = FLOW_PATH_URL_PATTERN.matcher(toStr(flowPathEntity.getAttribute(ATTR_URL)));
            if (urlMatcher.matches()) {
                flowPath.setGroupId(urlMatcher.group(1));
            }
        }
        flowPath.setExEntity(flowPathEntity);
        flowPath.setName(toStr(flowPathEntity.getAttribute(ATTR_NAME)));
        flowPath.getInputs().addAll(toQualifiedNameIds(toAtlasObjectIds(flowPathEntity.getAttribute(ATTR_INPUTS))).keySet());
        flowPath.getOutputs().addAll(toQualifiedNameIds(toAtlasObjectIds(flowPathEntity.getAttribute(ATTR_OUTPUTS))).keySet());
        flowPath.startTrackingChanges(nifiFlow);
        flowPaths.put(flowPath.getId(), flowPath);
    }
    nifiFlow.startTrackingChanges();
    return nifiFlow;
}

Example 2 with AtlasObjectId

use of org.apache.atlas.model.instance.AtlasObjectId in project nifi by apache.

the class NiFiAtlasClient method registerDataSetEntities.

/**
 * Register DataSet within specified NiFiFlow.
 * @return Set of registered Atlas type names and its remaining entities without deleted ones.
 */
private Map<String, List<AtlasEntity>> registerDataSetEntities(final NiFiFlow nifiFlow) throws AtlasServiceException {
    final Map<NiFiFlow.EntityChangeType, List<AtlasEntity>> changedEntities = nifiFlow.getChangedDataSetEntities();
    if (changedEntities.containsKey(CREATED)) {
        final List<AtlasEntity> createdEntities = changedEntities.get(CREATED);
        final AtlasEntity.AtlasEntitiesWithExtInfo atlasEntities = new AtlasEntity.AtlasEntitiesWithExtInfo(createdEntities);
        final EntityMutationResponse mutationResponse = atlasClient.createEntities(atlasEntities);
        logger.debug("Created DataSet entities mutation response={}", mutationResponse);
        final Map<String, String> guidAssignments = mutationResponse.getGuidAssignments();
        for (AtlasEntity entity : createdEntities) {
            final String guid = guidAssignments.get(entity.getGuid());
            final String qualifiedName = toStr(entity.getAttribute(ATTR_QUALIFIED_NAME));
            if (StringUtils.isEmpty(guid)) {
                logger.warn("GUID was not assigned for {}::{} for some reason.", entity.getTypeName(), qualifiedName);
                continue;
            }
            final Map<AtlasObjectId, AtlasEntity> entityMap;
            switch(entity.getTypeName()) {
                case TYPE_NIFI_INPUT_PORT:
                    entityMap = nifiFlow.getRootInputPortEntities();
                    break;
                case TYPE_NIFI_OUTPUT_PORT:
                    entityMap = nifiFlow.getRootOutputPortEntities();
                    break;
                case TYPE_NIFI_QUEUE:
                    entityMap = nifiFlow.getQueues();
                    break;
                default:
                    throw new RuntimeException(entity.getTypeName() + " is not expected.");
            }
            // In order to replace the id, remove current id which does not have GUID.
            findIdByQualifiedName(entityMap.keySet(), qualifiedName).ifPresent(entityMap::remove);
            entity.setGuid(guid);
            final AtlasObjectId idWithGuid = new AtlasObjectId(guid, entity.getTypeName(), Collections.singletonMap(ATTR_QUALIFIED_NAME, qualifiedName));
            entityMap.put(idWithGuid, entity);
        }
    }
    if (changedEntities.containsKey(UPDATED)) {
        final List<AtlasEntity> updatedEntities = changedEntities.get(UPDATED);
        final AtlasEntity.AtlasEntitiesWithExtInfo atlasEntities = new AtlasEntity.AtlasEntitiesWithExtInfo(updatedEntities);
        final EntityMutationResponse mutationResponse = atlasClient.updateEntities(atlasEntities);
        logger.debug("Updated DataSet entities mutation response={}", mutationResponse);
    }
    final Set<String> changedTypeNames = changedEntities.entrySet().stream().filter(entry -> !AS_IS.equals(entry.getKey())).flatMap(entry -> entry.getValue().stream()).map(AtlasEntity::getTypeName).collect(Collectors.toSet());
    // NOTE: Cascading DELETE will be performed when parent NiFiFlow is updated without removed DataSet entities.
    final Map<String, List<AtlasEntity>> remainingEntitiesByType = changedEntities.entrySet().stream().filter(entry -> !DELETED.equals(entry.getKey())).flatMap(entry -> entry.getValue().stream()).filter(entity -> changedTypeNames.contains(entity.getTypeName())).collect(Collectors.groupingBy(AtlasEntity::getTypeName));
    // If all entities are deleted for a type (e.g. nifi_intput_port), then remainingEntitiesByType will not contain such key.
    // If the returning map does not contain anything for a type, then the corresponding attribute will not be updated.
    // To empty an attribute when all of its elements are deleted, add empty list for a type.
    changedTypeNames.forEach(changedTypeName -> remainingEntitiesByType.computeIfAbsent(changedTypeName, k -> Collections.emptyList()));
    return remainingEntitiesByType;
}

Example 3 with AtlasObjectId

use of org.apache.atlas.model.instance.AtlasObjectId in project nifi by apache.

the class NiFiAtlasClient method registerFlowPathEntities.

private Set<AtlasObjectId> registerFlowPathEntities(final NiFiFlow nifiFlow) throws AtlasServiceException {
    final Map<NiFiFlow.EntityChangeType, List<AtlasEntity>> changedEntities = nifiFlow.getChangedFlowPathEntities();
    if (changedEntities.containsKey(CREATED)) {
        final List<AtlasEntity> createdEntities = changedEntities.get(CREATED);
        final AtlasEntity.AtlasEntitiesWithExtInfo atlasEntities = new AtlasEntity.AtlasEntitiesWithExtInfo(createdEntities);
        final EntityMutationResponse mutationResponse = atlasClient.createEntities(atlasEntities);
        logger.debug("Created FlowPath entities mutation response={}", mutationResponse);
        final Map<String, String> guidAssignments = mutationResponse.getGuidAssignments();
        createdEntities.forEach(entity -> {
            final String guid = entity.getGuid();
            entity.setGuid(guidAssignments.get(guid));
            final String pathId = getComponentIdFromQualifiedName(toStr(entity.getAttribute(ATTR_QUALIFIED_NAME)));
            final NiFiFlowPath path = nifiFlow.getFlowPaths().get(pathId);
            path.setExEntity(entity);
        });
    }
    if (changedEntities.containsKey(UPDATED)) {
        final List<AtlasEntity> updatedEntities = changedEntities.get(UPDATED);
        final AtlasEntity.AtlasEntitiesWithExtInfo atlasEntities = new AtlasEntity.AtlasEntitiesWithExtInfo(updatedEntities);
        final EntityMutationResponse mutationResponse = atlasClient.updateEntities(atlasEntities);
        logger.debug("Updated FlowPath entities mutation response={}", mutationResponse);
        updatedEntities.forEach(entity -> {
            final String pathId = getComponentIdFromQualifiedName(toStr(entity.getAttribute(ATTR_QUALIFIED_NAME)));
            final NiFiFlowPath path = nifiFlow.getFlowPaths().get(pathId);
            path.setExEntity(entity);
        });
    }
    if (NiFiFlow.EntityChangeType.containsChange(changedEntities.keySet())) {
        return changedEntities.entrySet().stream().filter(entry -> !DELETED.equals(entry.getKey())).flatMap(entry -> entry.getValue().stream()).map(path -> new AtlasObjectId(path.getGuid(), TYPE_NIFI_FLOW_PATH, Collections.singletonMap(ATTR_QUALIFIED_NAME, path.getAttribute(ATTR_QUALIFIED_NAME)))).collect(Collectors.toSet());
    }
    return null;
}

Example 4 with AtlasObjectId

use of org.apache.atlas.model.instance.AtlasObjectId in project nifi by apache.

the class TestNiFiFlowAnalyzer method testMultiPathsJoint.

@Test
public void testMultiPathsJoint() throws Exception {
    ProcessGroupStatus rootPG = createEmptyProcessGroupStatus();
    final ProcessorStatus pr0 = createProcessor(rootPG, "org.apache.nifi.processors.standard.GenerateFlowFile");
    final ProcessorStatus pr1 = createProcessor(rootPG, "org.apache.nifi.processors.standard.UpdateAttribute");
    final ProcessorStatus pr2 = createProcessor(rootPG, "org.apache.nifi.processors.standard.ListenTCP");
    final ProcessorStatus pr3 = createProcessor(rootPG, "org.apache.nifi.processors.standard.LogAttribute");
    // Result should be as follows:
    // pathA = 0 -> 1 (-> 3)
    // pathB = 2 (-> 3)
    // pathC = 3
    connect(rootPG, pr0, pr1);
    connect(rootPG, pr1, pr3);
    connect(rootPG, pr2, pr3);
    final NiFiFlowAnalyzer analyzer = new NiFiFlowAnalyzer();
    final NiFiFlow nifiFlow = new NiFiFlow(rootPG.getId());
    nifiFlow.setClusterName("cluster1");
    analyzer.analyzeProcessGroup(nifiFlow, rootPG);
    assertEquals(4, nifiFlow.getProcessors().size());
    analyzer.analyzePaths(nifiFlow);
    final Map<String, NiFiFlowPath> paths = nifiFlow.getFlowPaths();
    assertEquals(3, paths.size());
    // Order is not guaranteed
    final NiFiFlowPath pathA = paths.get(pr0.getId());
    final NiFiFlowPath pathB = paths.get(pr2.getId());
    final NiFiFlowPath pathC = paths.get(pr3.getId());
    assertEquals(2, pathA.getProcessComponentIds().size());
    assertEquals(1, pathB.getProcessComponentIds().size());
    assertEquals(1, pathC.getProcessComponentIds().size());
    // A queue is added as input for the joint point.
    assertEquals(1, pathC.getInputs().size());
    final AtlasObjectId queue = pathC.getInputs().iterator().next();
    assertEquals(TYPE_NIFI_QUEUE, queue.getTypeName());
    assertEquals(toQualifiedName("cluster1", pathC.getId()), queue.getUniqueAttributes().get(ATTR_QUALIFIED_NAME));
    // Should be able to find a path from a given processor GUID.
    final NiFiFlowPath pathForPr0 = nifiFlow.findPath(pr0.getId());
    final NiFiFlowPath pathForPr1 = nifiFlow.findPath(pr1.getId());
    final NiFiFlowPath pathForPr2 = nifiFlow.findPath(pr2.getId());
    final NiFiFlowPath pathForPr3 = nifiFlow.findPath(pr3.getId());
    assertEquals(pathA, pathForPr0);
    assertEquals(pathA, pathForPr1);
    assertEquals(pathB, pathForPr2);
    assertEquals(pathC, pathForPr3);
}

Example 5 with AtlasObjectId

use of org.apache.atlas.model.instance.AtlasObjectId in project atlas by apache.

the class EntityGraphMapper method mapAttributesAndClassifications.

public EntityMutationResponse mapAttributesAndClassifications(EntityMutationContext context, final boolean isPartialUpdate, final boolean replaceClassifications) throws AtlasBaseException {
    EntityMutationResponse resp = new EntityMutationResponse();
    Collection<AtlasEntity> createdEntities = context.getCreatedEntities();
    Collection<AtlasEntity> updatedEntities = context.getUpdatedEntities();
    if (CollectionUtils.isNotEmpty(createdEntities)) {
        for (AtlasEntity createdEntity : createdEntities) {
            String guid = createdEntity.getGuid();
            AtlasVertex vertex = context.getVertex(guid);
            AtlasEntityType entityType = context.getType(guid);
            compactAttributes(createdEntity);
            mapRelationshipAttributes(createdEntity, vertex, CREATE, context);
            mapAttributes(createdEntity, vertex, CREATE, context);
            resp.addEntity(CREATE, constructHeader(createdEntity, entityType, vertex));
            addClassifications(context, guid, createdEntity.getClassifications());
        }
    }
    if (CollectionUtils.isNotEmpty(updatedEntities)) {
        for (AtlasEntity updatedEntity : updatedEntities) {
            String guid = updatedEntity.getGuid();
            AtlasVertex vertex = context.getVertex(guid);
            AtlasEntityType entityType = context.getType(guid);
            compactAttributes(updatedEntity);
            mapRelationshipAttributes(updatedEntity, vertex, UPDATE, context);
            mapAttributes(updatedEntity, vertex, UPDATE, context);
            if (isPartialUpdate) {
                resp.addEntity(PARTIAL_UPDATE, constructHeader(updatedEntity, entityType, vertex));
            } else {
                resp.addEntity(UPDATE, constructHeader(updatedEntity, entityType, vertex));
            }
            if (replaceClassifications) {
                deleteClassifications(guid);
                addClassifications(context, guid, updatedEntity.getClassifications());
            }
        }
    }
    RequestContextV1 req = RequestContextV1.get();
    for (AtlasObjectId entity : req.getDeletedEntities()) {
        resp.addEntity(DELETE, entity);
    }
    for (AtlasObjectId entity : req.getUpdatedEntities()) {
        if (isPartialUpdate) {
            resp.addEntity(PARTIAL_UPDATE, entity);
        } else {
            resp.addEntity(UPDATE, entity);
        }
    }
    return resp;
}