use of org.apache.nifi.registry.flow.mapping.NiFiRegistryFlowMapper in project nifi by apache.
the class StandardNiFiServiceFacade method getLocalModifications.
@Override
public FlowComparisonEntity getLocalModifications(final String processGroupId) {
final ProcessGroup processGroup = processGroupDAO.getProcessGroup(processGroupId);
final VersionControlInformation versionControlInfo = processGroup.getVersionControlInformation();
if (versionControlInfo == null) {
throw new IllegalStateException("Process Group with ID " + processGroupId + " is not under Version Control");
}
final FlowRegistry flowRegistry = flowRegistryClient.getFlowRegistry(versionControlInfo.getRegistryIdentifier());
if (flowRegistry == null) {
throw new IllegalStateException("Process Group with ID " + processGroupId + " is tracking to a flow in Flow Registry with ID " + versionControlInfo.getRegistryIdentifier() + " but cannot find a Flow Registry with that identifier");
}
final VersionedFlowSnapshot versionedFlowSnapshot;
try {
versionedFlowSnapshot = flowRegistry.getFlowContents(versionControlInfo.getBucketIdentifier(), versionControlInfo.getFlowIdentifier(), versionControlInfo.getVersion(), true, NiFiUserUtils.getNiFiUser());
} catch (final IOException | NiFiRegistryException e) {
throw new NiFiCoreException("Failed to retrieve flow with Flow Registry in order to calculate local differences due to " + e.getMessage(), e);
}
final NiFiRegistryFlowMapper mapper = new NiFiRegistryFlowMapper();
final VersionedProcessGroup localGroup = mapper.mapProcessGroup(processGroup, controllerFacade.getControllerServiceProvider(), flowRegistryClient, true);
final VersionedProcessGroup registryGroup = versionedFlowSnapshot.getFlowContents();
final ComparableDataFlow localFlow = new StandardComparableDataFlow("Local Flow", localGroup);
final ComparableDataFlow registryFlow = new StandardComparableDataFlow("Versioned Flow", registryGroup);
final Set<String> ancestorServiceIds = getAncestorGroupServiceIds(processGroup);
final FlowComparator flowComparator = new StandardFlowComparator(registryFlow, localFlow, ancestorServiceIds, new ConciseEvolvingDifferenceDescriptor());
final FlowComparison flowComparison = flowComparator.compare();
final Set<ComponentDifferenceDTO> differenceDtos = dtoFactory.createComponentDifferenceDtos(flowComparison);
final FlowComparisonEntity entity = new FlowComparisonEntity();
entity.setComponentDifferences(differenceDtos);
return entity;
}
use of org.apache.nifi.registry.flow.mapping.NiFiRegistryFlowMapper in project nifi by apache.
the class StandardNiFiServiceFacade method createFlowSnapshot.
private InstantiatedVersionedProcessGroup createFlowSnapshot(final String processGroupId) {
final ProcessGroup processGroup = processGroupDAO.getProcessGroup(processGroupId);
final NiFiRegistryFlowMapper mapper = new NiFiRegistryFlowMapper();
final InstantiatedVersionedProcessGroup versionedGroup = mapper.mapProcessGroup(processGroup, controllerFacade.getControllerServiceProvider(), flowRegistryClient, false);
return versionedGroup;
}
use of org.apache.nifi.registry.flow.mapping.NiFiRegistryFlowMapper in project nifi by apache.
the class StandardProcessGroupDAO method updateVersionControlInformation.
@Override
public ProcessGroup updateVersionControlInformation(final VersionControlInformationDTO versionControlInformation, final Map<String, String> versionedComponentMapping) {
final String groupId = versionControlInformation.getGroupId();
final ProcessGroup group = locateProcessGroup(flowController, groupId);
final String registryId = versionControlInformation.getRegistryId();
final FlowRegistry flowRegistry = flowController.getFlowRegistryClient().getFlowRegistry(registryId);
final String registryName = flowRegistry == null ? registryId : flowRegistry.getName();
final NiFiRegistryFlowMapper mapper = new NiFiRegistryFlowMapper();
final VersionedProcessGroup flowSnapshot = mapper.mapProcessGroup(group, flowController, flowController.getFlowRegistryClient(), false);
final StandardVersionControlInformation vci = StandardVersionControlInformation.Builder.fromDto(versionControlInformation).registryName(registryName).flowSnapshot(flowSnapshot).build();
group.setVersionControlInformation(vci, versionedComponentMapping);
group.onComponentModified();
return group;
}
use of org.apache.nifi.registry.flow.mapping.NiFiRegistryFlowMapper in project nifi by apache.
the class StandardProcessGroup method updateFlow.
@Override
public void updateFlow(final VersionedFlowSnapshot proposedSnapshot, final String componentIdSeed, final boolean verifyNotDirty, final boolean updateSettings, final boolean updateDescendantVersionedFlows) {
writeLock.lock();
try {
verifyCanUpdate(proposedSnapshot, true, verifyNotDirty);
final NiFiRegistryFlowMapper mapper = new NiFiRegistryFlowMapper();
final VersionedProcessGroup versionedGroup = mapper.mapProcessGroup(this, controllerServiceProvider, flowController.getFlowRegistryClient(), true);
final ComparableDataFlow localFlow = new StandardComparableDataFlow("Local Flow", versionedGroup);
final ComparableDataFlow remoteFlow = new StandardComparableDataFlow("Remote Flow", proposedSnapshot.getFlowContents());
final FlowComparator flowComparator = new StandardFlowComparator(remoteFlow, localFlow, getAncestorGroupServiceIds(), new StaticDifferenceDescriptor());
final FlowComparison flowComparison = flowComparator.compare();
final Set<String> updatedVersionedComponentIds = new HashSet<>();
for (final FlowDifference diff : flowComparison.getDifferences()) {
// Ignore these as local differences for now because we can't do anything with it
if (diff.getDifferenceType() == DifferenceType.BUNDLE_CHANGED) {
continue;
}
// and if so compare our VersionedControllerService to the existing service.
if (diff.getDifferenceType() == DifferenceType.COMPONENT_ADDED) {
final VersionedComponent component = diff.getComponentA() == null ? diff.getComponentB() : diff.getComponentA();
if (ComponentType.CONTROLLER_SERVICE == component.getComponentType()) {
final ControllerServiceNode serviceNode = getVersionedControllerService(this, component.getIdentifier());
if (serviceNode != null) {
final VersionedControllerService versionedService = mapper.mapControllerService(serviceNode, controllerServiceProvider);
final Set<FlowDifference> differences = flowComparator.compareControllerServices(versionedService, (VersionedControllerService) component);
if (!differences.isEmpty()) {
updatedVersionedComponentIds.add(component.getIdentifier());
}
continue;
}
}
}
final VersionedComponent component = diff.getComponentA() == null ? diff.getComponentB() : diff.getComponentA();
updatedVersionedComponentIds.add(component.getIdentifier());
if (component.getComponentType() == ComponentType.REMOTE_INPUT_PORT || component.getComponentType() == ComponentType.REMOTE_OUTPUT_PORT) {
final String remoteGroupId = ((VersionedRemoteGroupPort) component).getRemoteGroupId();
updatedVersionedComponentIds.add(remoteGroupId);
}
}
if (LOG.isInfoEnabled()) {
final String differencesByLine = flowComparison.getDifferences().stream().map(FlowDifference::toString).collect(Collectors.joining("\n"));
LOG.info("Updating {} to {}; there are {} differences to take into account:\n{}", this, proposedSnapshot, flowComparison.getDifferences().size(), differencesByLine);
}
final Set<String> knownVariables = getKnownVariableNames();
final StandardVersionControlInformation originalVci = this.versionControlInfo.get();
try {
updateProcessGroup(this, proposedSnapshot.getFlowContents(), componentIdSeed, updatedVersionedComponentIds, false, updateSettings, updateDescendantVersionedFlows, knownVariables);
} catch (final Throwable t) {
// As a result, it is safe to use the get() and then the set() methods of the AtomicReference without introducing the 'check-then-modify' problem.
if (this.versionControlInfo.get() == null) {
this.versionControlInfo.set(originalVci);
}
throw t;
}
} catch (final ProcessorInstantiationException pie) {
throw new IllegalStateException("Failed to update flow", pie);
} finally {
writeLock.unlock();
}
}
use of org.apache.nifi.registry.flow.mapping.NiFiRegistryFlowMapper in project nifi by apache.
the class StandardNiFiServiceFacade method getComponentsAffectedByVersionChange.
@Override
public Set<AffectedComponentEntity> getComponentsAffectedByVersionChange(final String processGroupId, final VersionedFlowSnapshot updatedSnapshot, final NiFiUser user) {
final ProcessGroup group = processGroupDAO.getProcessGroup(processGroupId);
final NiFiRegistryFlowMapper mapper = new NiFiRegistryFlowMapper();
final VersionedProcessGroup localContents = mapper.mapProcessGroup(group, controllerFacade.getControllerServiceProvider(), flowRegistryClient, true);
final ComparableDataFlow localFlow = new StandardComparableDataFlow("Local Flow", localContents);
final ComparableDataFlow proposedFlow = new StandardComparableDataFlow("Versioned Flow", updatedSnapshot.getFlowContents());
final Set<String> ancestorGroupServiceIds = getAncestorGroupServiceIds(group);
final FlowComparator flowComparator = new StandardFlowComparator(localFlow, proposedFlow, ancestorGroupServiceIds, new StaticDifferenceDescriptor());
final FlowComparison comparison = flowComparator.compare();
final Set<AffectedComponentEntity> affectedComponents = comparison.getDifferences().stream().filter(// components that are added are not components that will be affected in the local flow.
difference -> difference.getDifferenceType() != DifferenceType.COMPONENT_ADDED).filter(difference -> difference.getDifferenceType() != DifferenceType.BUNDLE_CHANGED).filter(FlowDifferenceFilters.FILTER_ADDED_REMOVED_REMOTE_PORTS).map(difference -> {
final VersionedComponent localComponent = difference.getComponentA();
final String state;
switch(localComponent.getComponentType()) {
case CONTROLLER_SERVICE:
final String serviceId = ((InstantiatedVersionedControllerService) localComponent).getInstanceId();
state = controllerServiceDAO.getControllerService(serviceId).getState().name();
break;
case PROCESSOR:
final String processorId = ((InstantiatedVersionedProcessor) localComponent).getInstanceId();
state = processorDAO.getProcessor(processorId).getPhysicalScheduledState().name();
break;
case REMOTE_INPUT_PORT:
final InstantiatedVersionedRemoteGroupPort inputPort = (InstantiatedVersionedRemoteGroupPort) localComponent;
state = remoteProcessGroupDAO.getRemoteProcessGroup(inputPort.getInstanceGroupId()).getInputPort(inputPort.getInstanceId()).getScheduledState().name();
break;
case REMOTE_OUTPUT_PORT:
final InstantiatedVersionedRemoteGroupPort outputPort = (InstantiatedVersionedRemoteGroupPort) localComponent;
state = remoteProcessGroupDAO.getRemoteProcessGroup(outputPort.getInstanceGroupId()).getOutputPort(outputPort.getInstanceId()).getScheduledState().name();
break;
default:
state = null;
break;
}
return createAffectedComponentEntity((InstantiatedVersionedComponent) localComponent, localComponent.getComponentType().name(), state, user);
}).collect(Collectors.toCollection(HashSet::new));
for (final FlowDifference difference : comparison.getDifferences()) {
// Ignore these as local differences for now because we can't do anything with it
if (difference.getDifferenceType() == DifferenceType.BUNDLE_CHANGED) {
continue;
}
// Ignore differences for adding remote ports
if (FlowDifferenceFilters.isAddedOrRemovedRemotePort(difference)) {
continue;
}
final VersionedComponent localComponent = difference.getComponentA();
if (localComponent == null) {
continue;
}
// If any Process Group is removed, consider all components below that Process Group as an affected component
if (difference.getDifferenceType() == DifferenceType.COMPONENT_REMOVED && localComponent.getComponentType() == org.apache.nifi.registry.flow.ComponentType.PROCESS_GROUP) {
final String localGroupId = ((InstantiatedVersionedProcessGroup) localComponent).getInstanceId();
final ProcessGroup localGroup = processGroupDAO.getProcessGroup(localGroupId);
localGroup.findAllProcessors().stream().map(comp -> createAffectedComponentEntity(comp, user)).forEach(affectedComponents::add);
localGroup.findAllFunnels().stream().map(comp -> createAffectedComponentEntity(comp, user)).forEach(affectedComponents::add);
localGroup.findAllInputPorts().stream().map(comp -> createAffectedComponentEntity(comp, user)).forEach(affectedComponents::add);
localGroup.findAllOutputPorts().stream().map(comp -> createAffectedComponentEntity(comp, user)).forEach(affectedComponents::add);
localGroup.findAllRemoteProcessGroups().stream().flatMap(rpg -> Stream.concat(rpg.getInputPorts().stream(), rpg.getOutputPorts().stream())).map(comp -> createAffectedComponentEntity(comp, user)).forEach(affectedComponents::add);
localGroup.findAllControllerServices().stream().map(comp -> createAffectedComponentEntity(comp, user)).forEach(affectedComponents::add);
}
if (localComponent.getComponentType() == org.apache.nifi.registry.flow.ComponentType.CONTROLLER_SERVICE) {
final String serviceId = ((InstantiatedVersionedControllerService) localComponent).getInstanceId();
final ControllerServiceNode serviceNode = controllerServiceDAO.getControllerService(serviceId);
final List<ControllerServiceNode> referencingServices = serviceNode.getReferences().findRecursiveReferences(ControllerServiceNode.class);
for (final ControllerServiceNode referencingService : referencingServices) {
affectedComponents.add(createAffectedComponentEntity(referencingService, user));
}
final List<ProcessorNode> referencingProcessors = serviceNode.getReferences().findRecursiveReferences(ProcessorNode.class);
for (final ProcessorNode referencingProcessor : referencingProcessors) {
affectedComponents.add(createAffectedComponentEntity(referencingProcessor, user));
}
}
}
// Create a map of all connectable components by versioned component ID to the connectable component itself
final Map<String, List<Connectable>> connectablesByVersionId = new HashMap<>();
mapToConnectableId(group.findAllFunnels(), connectablesByVersionId);
mapToConnectableId(group.findAllInputPorts(), connectablesByVersionId);
mapToConnectableId(group.findAllOutputPorts(), connectablesByVersionId);
mapToConnectableId(group.findAllProcessors(), connectablesByVersionId);
final List<RemoteGroupPort> remotePorts = new ArrayList<>();
for (final RemoteProcessGroup rpg : group.findAllRemoteProcessGroups()) {
remotePorts.addAll(rpg.getInputPorts());
remotePorts.addAll(rpg.getOutputPorts());
}
mapToConnectableId(remotePorts, connectablesByVersionId);
// and the destination (if it exists in the flow currently).
for (final FlowDifference difference : comparison.getDifferences()) {
VersionedComponent component = difference.getComponentA();
if (component == null) {
component = difference.getComponentB();
}
if (component.getComponentType() != org.apache.nifi.registry.flow.ComponentType.CONNECTION) {
continue;
}
final VersionedConnection connection = (VersionedConnection) component;
final String sourceVersionedId = connection.getSource().getId();
final List<Connectable> sources = connectablesByVersionId.get(sourceVersionedId);
if (sources != null) {
for (final Connectable source : sources) {
affectedComponents.add(createAffectedComponentEntity(source, user));
}
}
final String destinationVersionId = connection.getDestination().getId();
final List<Connectable> destinations = connectablesByVersionId.get(destinationVersionId);
if (destinations != null) {
for (final Connectable destination : destinations) {
affectedComponents.add(createAffectedComponentEntity(destination, user));
}
}
}
return affectedComponents;
}
Aggregations