Examples with ComparableDataFlow org.apache.nifi.registry.flow.diff.ComparableDataFlow used on opensource projects

Example 1 with ComparableDataFlow

use of org.apache.nifi.registry.flow.diff.ComparableDataFlow in project nifi by apache.

the class StandardNiFiServiceFacade method getLocalModifications.

@Override
public FlowComparisonEntity getLocalModifications(final String processGroupId) {
    final ProcessGroup processGroup = processGroupDAO.getProcessGroup(processGroupId);
    final VersionControlInformation versionControlInfo = processGroup.getVersionControlInformation();
    if (versionControlInfo == null) {
        throw new IllegalStateException("Process Group with ID " + processGroupId + " is not under Version Control");
    }
    final FlowRegistry flowRegistry = flowRegistryClient.getFlowRegistry(versionControlInfo.getRegistryIdentifier());
    if (flowRegistry == null) {
        throw new IllegalStateException("Process Group with ID " + processGroupId + " is tracking to a flow in Flow Registry with ID " + versionControlInfo.getRegistryIdentifier() + " but cannot find a Flow Registry with that identifier");
    }
    final VersionedFlowSnapshot versionedFlowSnapshot;
    try {
        versionedFlowSnapshot = flowRegistry.getFlowContents(versionControlInfo.getBucketIdentifier(), versionControlInfo.getFlowIdentifier(), versionControlInfo.getVersion(), true, NiFiUserUtils.getNiFiUser());
    } catch (final IOException | NiFiRegistryException e) {
        throw new NiFiCoreException("Failed to retrieve flow with Flow Registry in order to calculate local differences due to " + e.getMessage(), e);
    }
    final NiFiRegistryFlowMapper mapper = new NiFiRegistryFlowMapper();
    final VersionedProcessGroup localGroup = mapper.mapProcessGroup(processGroup, controllerFacade.getControllerServiceProvider(), flowRegistryClient, true);
    final VersionedProcessGroup registryGroup = versionedFlowSnapshot.getFlowContents();
    final ComparableDataFlow localFlow = new StandardComparableDataFlow("Local Flow", localGroup);
    final ComparableDataFlow registryFlow = new StandardComparableDataFlow("Versioned Flow", registryGroup);
    final Set<String> ancestorServiceIds = getAncestorGroupServiceIds(processGroup);
    final FlowComparator flowComparator = new StandardFlowComparator(registryFlow, localFlow, ancestorServiceIds, new ConciseEvolvingDifferenceDescriptor());
    final FlowComparison flowComparison = flowComparator.compare();
    final Set<ComponentDifferenceDTO> differenceDtos = dtoFactory.createComponentDifferenceDtos(flowComparison);
    final FlowComparisonEntity entity = new FlowComparisonEntity();
    entity.setComponentDifferences(differenceDtos);
    return entity;
}

Example 2 with ComparableDataFlow

use of org.apache.nifi.registry.flow.diff.ComparableDataFlow in project nifi by apache.

the class StandardProcessGroup method updateFlow.

@Override
public void updateFlow(final VersionedFlowSnapshot proposedSnapshot, final String componentIdSeed, final boolean verifyNotDirty, final boolean updateSettings, final boolean updateDescendantVersionedFlows) {
    writeLock.lock();
    try {
        verifyCanUpdate(proposedSnapshot, true, verifyNotDirty);
        final NiFiRegistryFlowMapper mapper = new NiFiRegistryFlowMapper();
        final VersionedProcessGroup versionedGroup = mapper.mapProcessGroup(this, controllerServiceProvider, flowController.getFlowRegistryClient(), true);
        final ComparableDataFlow localFlow = new StandardComparableDataFlow("Local Flow", versionedGroup);
        final ComparableDataFlow remoteFlow = new StandardComparableDataFlow("Remote Flow", proposedSnapshot.getFlowContents());
        final FlowComparator flowComparator = new StandardFlowComparator(remoteFlow, localFlow, getAncestorGroupServiceIds(), new StaticDifferenceDescriptor());
        final FlowComparison flowComparison = flowComparator.compare();
        final Set<String> updatedVersionedComponentIds = new HashSet<>();
        for (final FlowDifference diff : flowComparison.getDifferences()) {
            // Ignore these as local differences for now because we can't do anything with it
            if (diff.getDifferenceType() == DifferenceType.BUNDLE_CHANGED) {
                continue;
            }
            // and if so compare our VersionedControllerService to the existing service.
            if (diff.getDifferenceType() == DifferenceType.COMPONENT_ADDED) {
                final VersionedComponent component = diff.getComponentA() == null ? diff.getComponentB() : diff.getComponentA();
                if (ComponentType.CONTROLLER_SERVICE == component.getComponentType()) {
                    final ControllerServiceNode serviceNode = getVersionedControllerService(this, component.getIdentifier());
                    if (serviceNode != null) {
                        final VersionedControllerService versionedService = mapper.mapControllerService(serviceNode, controllerServiceProvider);
                        final Set<FlowDifference> differences = flowComparator.compareControllerServices(versionedService, (VersionedControllerService) component);
                        if (!differences.isEmpty()) {
                            updatedVersionedComponentIds.add(component.getIdentifier());
                        }
                        continue;
                    }
                }
            }
            final VersionedComponent component = diff.getComponentA() == null ? diff.getComponentB() : diff.getComponentA();
            updatedVersionedComponentIds.add(component.getIdentifier());
            if (component.getComponentType() == ComponentType.REMOTE_INPUT_PORT || component.getComponentType() == ComponentType.REMOTE_OUTPUT_PORT) {
                final String remoteGroupId = ((VersionedRemoteGroupPort) component).getRemoteGroupId();
                updatedVersionedComponentIds.add(remoteGroupId);
            }
        }
        if (LOG.isInfoEnabled()) {
            final String differencesByLine = flowComparison.getDifferences().stream().map(FlowDifference::toString).collect(Collectors.joining("\n"));
            LOG.info("Updating {} to {}; there are {} differences to take into account:\n{}", this, proposedSnapshot, flowComparison.getDifferences().size(), differencesByLine);
        }
        final Set<String> knownVariables = getKnownVariableNames();
        final StandardVersionControlInformation originalVci = this.versionControlInfo.get();
        try {
            updateProcessGroup(this, proposedSnapshot.getFlowContents(), componentIdSeed, updatedVersionedComponentIds, false, updateSettings, updateDescendantVersionedFlows, knownVariables);
        } catch (final Throwable t) {
            // As a result, it is safe to use the get() and then the set() methods of the AtomicReference without introducing the 'check-then-modify' problem.
            if (this.versionControlInfo.get() == null) {
                this.versionControlInfo.set(originalVci);
            }
            throw t;
        }
    } catch (final ProcessorInstantiationException pie) {
        throw new IllegalStateException("Failed to update flow", pie);
    } finally {
        writeLock.unlock();
    }
}

Example 3 with ComparableDataFlow

use of org.apache.nifi.registry.flow.diff.ComparableDataFlow in project nifi-registry by apache.

the class RegistryService method getFlowDiff.

/**
 * Returns the differences between two specified versions of a flow.
 *
 * @param bucketIdentifier the id of the bucket the flow exists in
 * @param flowIdentifier the flow to be examined
 * @param versionA the first version of the comparison
 * @param versionB the second version of the comparison
 * @return The differences between two specified versions, grouped by component.
 */
public VersionedFlowDifference getFlowDiff(final String bucketIdentifier, final String flowIdentifier, final Integer versionA, final Integer versionB) {
    if (StringUtils.isBlank(bucketIdentifier)) {
        throw new IllegalArgumentException("Bucket identifier cannot be null or blank");
    }
    if (StringUtils.isBlank(flowIdentifier)) {
        throw new IllegalArgumentException("Flow identifier cannot be null or blank");
    }
    if (versionA == null || versionB == null) {
        throw new IllegalArgumentException("Version cannot be null or blank");
    }
    // older version is always the lower, regardless of the order supplied
    final Integer older = Math.min(versionA, versionB);
    final Integer newer = Math.max(versionA, versionB);
    readLock.lock();
    try {
        // Get the content for both versions of the flow
        final byte[] serializedSnapshotA = flowPersistenceProvider.getFlowContent(bucketIdentifier, flowIdentifier, older);
        if (serializedSnapshotA == null || serializedSnapshotA.length == 0) {
            throw new IllegalStateException("No serialized content found for snapshot with flow identifier " + flowIdentifier + " and version " + older);
        }
        final byte[] serializedSnapshotB = flowPersistenceProvider.getFlowContent(bucketIdentifier, flowIdentifier, newer);
        if (serializedSnapshotB == null || serializedSnapshotB.length == 0) {
            throw new IllegalStateException("No serialized content found for snapshot with flow identifier " + flowIdentifier + " and version " + newer);
        }
        // deserialize the contents
        final InputStream inputA = new ByteArrayInputStream(serializedSnapshotA);
        final VersionedProcessGroup flowContentsA = processGroupSerializer.deserialize(inputA);
        final InputStream inputB = new ByteArrayInputStream(serializedSnapshotB);
        final VersionedProcessGroup flowContentsB = processGroupSerializer.deserialize(inputB);
        final ComparableDataFlow comparableFlowA = new StandardComparableDataFlow(String.format("Version %d", older), flowContentsA);
        final ComparableDataFlow comparableFlowB = new StandardComparableDataFlow(String.format("Version %d", newer), flowContentsB);
        // Compare the two versions of the flow
        final FlowComparator flowComparator = new StandardFlowComparator(comparableFlowA, comparableFlowB, null, new ConciseEvolvingDifferenceDescriptor());
        final FlowComparison flowComparison = flowComparator.compare();
        VersionedFlowDifference result = new VersionedFlowDifference();
        result.setBucketId(bucketIdentifier);
        result.setFlowId(flowIdentifier);
        result.setVersionA(older);
        result.setVersionB(newer);
        Set<ComponentDifferenceGroup> differenceGroups = getStringComponentDifferenceGroupMap(flowComparison.getDifferences());
        result.setComponentDifferenceGroups(differenceGroups);
        return result;
    } finally {
        readLock.unlock();
    }
}

Example 4 with ComparableDataFlow

use of org.apache.nifi.registry.flow.diff.ComparableDataFlow in project nifi by apache.

the class StandardNiFiServiceFacade method getComponentsAffectedByVersionChange.

@Override
public Set<AffectedComponentEntity> getComponentsAffectedByVersionChange(final String processGroupId, final VersionedFlowSnapshot updatedSnapshot, final NiFiUser user) {
    final ProcessGroup group = processGroupDAO.getProcessGroup(processGroupId);
    final NiFiRegistryFlowMapper mapper = new NiFiRegistryFlowMapper();
    final VersionedProcessGroup localContents = mapper.mapProcessGroup(group, controllerFacade.getControllerServiceProvider(), flowRegistryClient, true);
    final ComparableDataFlow localFlow = new StandardComparableDataFlow("Local Flow", localContents);
    final ComparableDataFlow proposedFlow = new StandardComparableDataFlow("Versioned Flow", updatedSnapshot.getFlowContents());
    final Set<String> ancestorGroupServiceIds = getAncestorGroupServiceIds(group);
    final FlowComparator flowComparator = new StandardFlowComparator(localFlow, proposedFlow, ancestorGroupServiceIds, new StaticDifferenceDescriptor());
    final FlowComparison comparison = flowComparator.compare();
    final Set<AffectedComponentEntity> affectedComponents = comparison.getDifferences().stream().filter(// components that are added are not components that will be affected in the local flow.
    difference -> difference.getDifferenceType() != DifferenceType.COMPONENT_ADDED).filter(difference -> difference.getDifferenceType() != DifferenceType.BUNDLE_CHANGED).filter(FlowDifferenceFilters.FILTER_ADDED_REMOVED_REMOTE_PORTS).map(difference -> {
        final VersionedComponent localComponent = difference.getComponentA();
        final String state;
        switch(localComponent.getComponentType()) {
            case CONTROLLER_SERVICE:
                final String serviceId = ((InstantiatedVersionedControllerService) localComponent).getInstanceId();
                state = controllerServiceDAO.getControllerService(serviceId).getState().name();
                break;
            case PROCESSOR:
                final String processorId = ((InstantiatedVersionedProcessor) localComponent).getInstanceId();
                state = processorDAO.getProcessor(processorId).getPhysicalScheduledState().name();
                break;
            case REMOTE_INPUT_PORT:
                final InstantiatedVersionedRemoteGroupPort inputPort = (InstantiatedVersionedRemoteGroupPort) localComponent;
                state = remoteProcessGroupDAO.getRemoteProcessGroup(inputPort.getInstanceGroupId()).getInputPort(inputPort.getInstanceId()).getScheduledState().name();
                break;
            case REMOTE_OUTPUT_PORT:
                final InstantiatedVersionedRemoteGroupPort outputPort = (InstantiatedVersionedRemoteGroupPort) localComponent;
                state = remoteProcessGroupDAO.getRemoteProcessGroup(outputPort.getInstanceGroupId()).getOutputPort(outputPort.getInstanceId()).getScheduledState().name();
                break;
            default:
                state = null;
                break;
        }
        return createAffectedComponentEntity((InstantiatedVersionedComponent) localComponent, localComponent.getComponentType().name(), state, user);
    }).collect(Collectors.toCollection(HashSet::new));
    for (final FlowDifference difference : comparison.getDifferences()) {
        // Ignore these as local differences for now because we can't do anything with it
        if (difference.getDifferenceType() == DifferenceType.BUNDLE_CHANGED) {
            continue;
        }
        // Ignore differences for adding remote ports
        if (FlowDifferenceFilters.isAddedOrRemovedRemotePort(difference)) {
            continue;
        }
        final VersionedComponent localComponent = difference.getComponentA();
        if (localComponent == null) {
            continue;
        }
        // If any Process Group is removed, consider all components below that Process Group as an affected component
        if (difference.getDifferenceType() == DifferenceType.COMPONENT_REMOVED && localComponent.getComponentType() == org.apache.nifi.registry.flow.ComponentType.PROCESS_GROUP) {
            final String localGroupId = ((InstantiatedVersionedProcessGroup) localComponent).getInstanceId();
            final ProcessGroup localGroup = processGroupDAO.getProcessGroup(localGroupId);
            localGroup.findAllProcessors().stream().map(comp -> createAffectedComponentEntity(comp, user)).forEach(affectedComponents::add);
            localGroup.findAllFunnels().stream().map(comp -> createAffectedComponentEntity(comp, user)).forEach(affectedComponents::add);
            localGroup.findAllInputPorts().stream().map(comp -> createAffectedComponentEntity(comp, user)).forEach(affectedComponents::add);
            localGroup.findAllOutputPorts().stream().map(comp -> createAffectedComponentEntity(comp, user)).forEach(affectedComponents::add);
            localGroup.findAllRemoteProcessGroups().stream().flatMap(rpg -> Stream.concat(rpg.getInputPorts().stream(), rpg.getOutputPorts().stream())).map(comp -> createAffectedComponentEntity(comp, user)).forEach(affectedComponents::add);
            localGroup.findAllControllerServices().stream().map(comp -> createAffectedComponentEntity(comp, user)).forEach(affectedComponents::add);
        }
        if (localComponent.getComponentType() == org.apache.nifi.registry.flow.ComponentType.CONTROLLER_SERVICE) {
            final String serviceId = ((InstantiatedVersionedControllerService) localComponent).getInstanceId();
            final ControllerServiceNode serviceNode = controllerServiceDAO.getControllerService(serviceId);
            final List<ControllerServiceNode> referencingServices = serviceNode.getReferences().findRecursiveReferences(ControllerServiceNode.class);
            for (final ControllerServiceNode referencingService : referencingServices) {
                affectedComponents.add(createAffectedComponentEntity(referencingService, user));
            }
            final List<ProcessorNode> referencingProcessors = serviceNode.getReferences().findRecursiveReferences(ProcessorNode.class);
            for (final ProcessorNode referencingProcessor : referencingProcessors) {
                affectedComponents.add(createAffectedComponentEntity(referencingProcessor, user));
            }
        }
    }
    // Create a map of all connectable components by versioned component ID to the connectable component itself
    final Map<String, List<Connectable>> connectablesByVersionId = new HashMap<>();
    mapToConnectableId(group.findAllFunnels(), connectablesByVersionId);
    mapToConnectableId(group.findAllInputPorts(), connectablesByVersionId);
    mapToConnectableId(group.findAllOutputPorts(), connectablesByVersionId);
    mapToConnectableId(group.findAllProcessors(), connectablesByVersionId);
    final List<RemoteGroupPort> remotePorts = new ArrayList<>();
    for (final RemoteProcessGroup rpg : group.findAllRemoteProcessGroups()) {
        remotePorts.addAll(rpg.getInputPorts());
        remotePorts.addAll(rpg.getOutputPorts());
    }
    mapToConnectableId(remotePorts, connectablesByVersionId);
    // and the destination (if it exists in the flow currently).
    for (final FlowDifference difference : comparison.getDifferences()) {
        VersionedComponent component = difference.getComponentA();
        if (component == null) {
            component = difference.getComponentB();
        }
        if (component.getComponentType() != org.apache.nifi.registry.flow.ComponentType.CONNECTION) {
            continue;
        }
        final VersionedConnection connection = (VersionedConnection) component;
        final String sourceVersionedId = connection.getSource().getId();
        final List<Connectable> sources = connectablesByVersionId.get(sourceVersionedId);
        if (sources != null) {
            for (final Connectable source : sources) {
                affectedComponents.add(createAffectedComponentEntity(source, user));
            }
        }
        final String destinationVersionId = connection.getDestination().getId();
        final List<Connectable> destinations = connectablesByVersionId.get(destinationVersionId);
        if (destinations != null) {
            for (final Connectable destination : destinations) {
                affectedComponents.add(createAffectedComponentEntity(destination, user));
            }
        }
    }
    return affectedComponents;
}

Example 5 with ComparableDataFlow

use of org.apache.nifi.registry.flow.diff.ComparableDataFlow in project nifi by apache.

the class StandardProcessGroup method getModifications.

private Set<FlowDifference> getModifications() {
    final StandardVersionControlInformation vci = versionControlInfo.get();
    // So we have to perform a diff of the flows and see if they are the same.
    if (vci == null) {
        return null;
    }
    if (vci.getFlowSnapshot() == null) {
        // As a result, we will just return an empty optional
        return null;
    }
    final NiFiRegistryFlowMapper mapper = new NiFiRegistryFlowMapper();
    final VersionedProcessGroup versionedGroup = mapper.mapProcessGroup(this, controllerServiceProvider, flowController.getFlowRegistryClient(), false);
    final ComparableDataFlow currentFlow = new StandardComparableDataFlow("Local Flow", versionedGroup);
    final ComparableDataFlow snapshotFlow = new StandardComparableDataFlow("Versioned Flow", vci.getFlowSnapshot());
    final FlowComparator flowComparator = new StandardFlowComparator(snapshotFlow, currentFlow, getAncestorGroupServiceIds(), new EvolvingDifferenceDescriptor());
    final FlowComparison comparison = flowComparator.compare();
    final Set<FlowDifference> differences = comparison.getDifferences().stream().filter(difference -> difference.getDifferenceType() != DifferenceType.BUNDLE_CHANGED).filter(FlowDifferenceFilters.FILTER_ADDED_REMOVED_REMOTE_PORTS).collect(Collectors.toCollection(HashSet::new));
    LOG.debug("There are {} differences between this Local Flow and the Versioned Flow: {}", differences.size(), differences);
    return differences;
}