Examples with VersionedProcessGroup org.apache.nifi.registry.flow.VersionedProcessGroup used on opensource projects

Example 1 with VersionedProcessGroup

use of org.apache.nifi.registry.flow.VersionedProcessGroup in project nifi by apache.

the class StandardNiFiServiceFacade method getLocalModifications.

@Override
public FlowComparisonEntity getLocalModifications(final String processGroupId) {
    final ProcessGroup processGroup = processGroupDAO.getProcessGroup(processGroupId);
    final VersionControlInformation versionControlInfo = processGroup.getVersionControlInformation();
    if (versionControlInfo == null) {
        throw new IllegalStateException("Process Group with ID " + processGroupId + " is not under Version Control");
    }
    final FlowRegistry flowRegistry = flowRegistryClient.getFlowRegistry(versionControlInfo.getRegistryIdentifier());
    if (flowRegistry == null) {
        throw new IllegalStateException("Process Group with ID " + processGroupId + " is tracking to a flow in Flow Registry with ID " + versionControlInfo.getRegistryIdentifier() + " but cannot find a Flow Registry with that identifier");
    }
    final VersionedFlowSnapshot versionedFlowSnapshot;
    try {
        versionedFlowSnapshot = flowRegistry.getFlowContents(versionControlInfo.getBucketIdentifier(), versionControlInfo.getFlowIdentifier(), versionControlInfo.getVersion(), true, NiFiUserUtils.getNiFiUser());
    } catch (final IOException | NiFiRegistryException e) {
        throw new NiFiCoreException("Failed to retrieve flow with Flow Registry in order to calculate local differences due to " + e.getMessage(), e);
    }
    final NiFiRegistryFlowMapper mapper = new NiFiRegistryFlowMapper();
    final VersionedProcessGroup localGroup = mapper.mapProcessGroup(processGroup, controllerFacade.getControllerServiceProvider(), flowRegistryClient, true);
    final VersionedProcessGroup registryGroup = versionedFlowSnapshot.getFlowContents();
    final ComparableDataFlow localFlow = new StandardComparableDataFlow("Local Flow", localGroup);
    final ComparableDataFlow registryFlow = new StandardComparableDataFlow("Versioned Flow", registryGroup);
    final Set<String> ancestorServiceIds = getAncestorGroupServiceIds(processGroup);
    final FlowComparator flowComparator = new StandardFlowComparator(registryFlow, localFlow, ancestorServiceIds, new ConciseEvolvingDifferenceDescriptor());
    final FlowComparison flowComparison = flowComparator.compare();
    final Set<ComponentDifferenceDTO> differenceDtos = dtoFactory.createComponentDifferenceDtos(flowComparison);
    final FlowComparisonEntity entity = new FlowComparisonEntity();
    entity.setComponentDifferences(differenceDtos);
    return entity;
}

Example 2 with VersionedProcessGroup

use of org.apache.nifi.registry.flow.VersionedProcessGroup in project nifi by apache.

the class StandardNiFiServiceFacade method registerFlowWithFlowRegistry.

@Override
public VersionControlComponentMappingEntity registerFlowWithFlowRegistry(final String groupId, final StartVersionControlRequestEntity requestEntity) {
    final ProcessGroup processGroup = processGroupDAO.getProcessGroup(groupId);
    final VersionControlInformation currentVci = processGroup.getVersionControlInformation();
    final int expectedVersion = currentVci == null ? 1 : currentVci.getVersion() + 1;
    // Create a VersionedProcessGroup snapshot of the flow as it is currently.
    final InstantiatedVersionedProcessGroup versionedProcessGroup = createFlowSnapshot(groupId);
    final VersionedFlowDTO versionedFlowDto = requestEntity.getVersionedFlow();
    final String flowId = versionedFlowDto.getFlowId() == null ? UUID.randomUUID().toString() : versionedFlowDto.getFlowId();
    final VersionedFlow versionedFlow = new VersionedFlow();
    versionedFlow.setBucketIdentifier(versionedFlowDto.getBucketId());
    versionedFlow.setCreatedTimestamp(System.currentTimeMillis());
    versionedFlow.setDescription(versionedFlowDto.getDescription());
    versionedFlow.setModifiedTimestamp(versionedFlow.getCreatedTimestamp());
    versionedFlow.setName(versionedFlowDto.getFlowName());
    versionedFlow.setIdentifier(flowId);
    // Add the Versioned Flow and first snapshot to the Flow Registry
    final String registryId = requestEntity.getVersionedFlow().getRegistryId();
    final VersionedFlowSnapshot registeredSnapshot;
    final VersionedFlow registeredFlow;
    String action = "create the flow";
    try {
        // first, create the flow in the registry, if necessary
        if (versionedFlowDto.getFlowId() == null) {
            registeredFlow = registerVersionedFlow(registryId, versionedFlow);
        } else {
            registeredFlow = getVersionedFlow(registryId, versionedFlowDto.getBucketId(), versionedFlowDto.getFlowId());
        }
        action = "add the local flow to the Flow Registry as the first Snapshot";
        // add first snapshot to the flow in the registry
        registeredSnapshot = registerVersionedFlowSnapshot(registryId, registeredFlow, versionedProcessGroup, versionedFlowDto.getComments(), expectedVersion);
    } catch (final NiFiRegistryException e) {
        throw new IllegalArgumentException(e.getLocalizedMessage());
    } catch (final IOException ioe) {
        throw new IllegalStateException("Failed to communicate with Flow Registry when attempting to " + action);
    }
    final Bucket bucket = registeredSnapshot.getBucket();
    final VersionedFlow flow = registeredSnapshot.getFlow();
    // Update the Process Group with the new VersionControlInformation. (Send this to all nodes).
    final VersionControlInformationDTO vci = new VersionControlInformationDTO();
    vci.setBucketId(bucket.getIdentifier());
    vci.setBucketName(bucket.getName());
    vci.setFlowId(flow.getIdentifier());
    vci.setFlowName(flow.getName());
    vci.setFlowDescription(flow.getDescription());
    vci.setGroupId(groupId);
    vci.setRegistryId(registryId);
    vci.setRegistryName(getFlowRegistryName(registryId));
    vci.setVersion(registeredSnapshot.getSnapshotMetadata().getVersion());
    vci.setState(VersionedFlowState.UP_TO_DATE.name());
    final Map<String, String> mapping = dtoFactory.createVersionControlComponentMappingDto(versionedProcessGroup);
    final Revision groupRevision = revisionManager.getRevision(groupId);
    final RevisionDTO groupRevisionDto = dtoFactory.createRevisionDTO(groupRevision);
    final VersionControlComponentMappingEntity entity = new VersionControlComponentMappingEntity();
    entity.setVersionControlInformation(vci);
    entity.setProcessGroupRevision(groupRevisionDto);
    entity.setVersionControlComponentMapping(mapping);
    return entity;
}

Example 3 with VersionedProcessGroup

use of org.apache.nifi.registry.flow.VersionedProcessGroup in project nifi by apache.

the class StandardProcessGroupDAO method updateVersionControlInformation.

@Override
public ProcessGroup updateVersionControlInformation(final VersionControlInformationDTO versionControlInformation, final Map<String, String> versionedComponentMapping) {
    final String groupId = versionControlInformation.getGroupId();
    final ProcessGroup group = locateProcessGroup(flowController, groupId);
    final String registryId = versionControlInformation.getRegistryId();
    final FlowRegistry flowRegistry = flowController.getFlowRegistryClient().getFlowRegistry(registryId);
    final String registryName = flowRegistry == null ? registryId : flowRegistry.getName();
    final NiFiRegistryFlowMapper mapper = new NiFiRegistryFlowMapper();
    final VersionedProcessGroup flowSnapshot = mapper.mapProcessGroup(group, flowController, flowController.getFlowRegistryClient(), false);
    final StandardVersionControlInformation vci = StandardVersionControlInformation.Builder.fromDto(versionControlInformation).registryName(registryName).flowSnapshot(flowSnapshot).build();
    group.setVersionControlInformation(vci, versionedComponentMapping);
    group.onComponentModified();
    return group;
}

Example 4 with VersionedProcessGroup

use of org.apache.nifi.registry.flow.VersionedProcessGroup in project nifi by apache.

the class StandardProcessGroup method synchronizeWithFlowRegistry.

@Override
public void synchronizeWithFlowRegistry(final FlowRegistryClient flowRegistryClient) {
    final StandardVersionControlInformation vci = versionControlInfo.get();
    if (vci == null) {
        return;
    }
    final String registryId = vci.getRegistryIdentifier();
    final FlowRegistry flowRegistry = flowRegistryClient.getFlowRegistry(registryId);
    if (flowRegistry == null) {
        final String message = String.format("Unable to synchronize Process Group with Flow Registry because Process Group was placed under Version Control using Flow Registry " + "with identifier %s but cannot find any Flow Registry with this identifier", registryId);
        versionControlFields.setSyncFailureExplanation(message);
        LOG.error("Unable to synchronize {} with Flow Registry because Process Group was placed under Version Control using Flow Registry " + "with identifier {} but cannot find any Flow Registry with this identifier", this, registryId);
        return;
    }
    final VersionedProcessGroup snapshot = vci.getFlowSnapshot();
    if (snapshot == null) {
        // This allows us to know whether or not the flow has been modified since it was last synced with the Flow Registry.
        try {
            final VersionedFlowSnapshot registrySnapshot = flowRegistry.getFlowContents(vci.getBucketIdentifier(), vci.getFlowIdentifier(), vci.getVersion(), false);
            final VersionedProcessGroup registryFlow = registrySnapshot.getFlowContents();
            vci.setFlowSnapshot(registryFlow);
        } catch (final IOException | NiFiRegistryException e) {
            final String message = String.format("Failed to synchronize Process Group with Flow Registry because could not retrieve version %s of flow with identifier %s in bucket %s", vci.getVersion(), vci.getFlowIdentifier(), vci.getBucketIdentifier());
            versionControlFields.setSyncFailureExplanation(message);
            LOG.error("Failed to synchronize {} with Flow Registry because could not retrieve version {} of flow with identifier {} in bucket {}", this, vci.getVersion(), vci.getFlowIdentifier(), vci.getBucketIdentifier(), e);
            return;
        }
    }
    try {
        final VersionedFlow versionedFlow = flowRegistry.getVersionedFlow(vci.getBucketIdentifier(), vci.getFlowIdentifier());
        final int latestVersion = (int) versionedFlow.getVersionCount();
        vci.setBucketName(versionedFlow.getBucketName());
        vci.setFlowName(versionedFlow.getName());
        vci.setFlowDescription(versionedFlow.getDescription());
        vci.setRegistryName(flowRegistry.getName());
        if (latestVersion == vci.getVersion()) {
            LOG.debug("{} is currently at the most recent version ({}) of the flow that is under Version Control", this, latestVersion);
            versionControlFields.setStale(false);
        } else {
            LOG.info("{} is not the most recent version of the flow that is under Version Control; current version is {}; most recent version is {}", new Object[] { this, vci.getVersion(), latestVersion });
            versionControlFields.setStale(true);
        }
        versionControlFields.setSyncFailureExplanation(null);
    } catch (final IOException | NiFiRegistryException e) {
        final String message = String.format("Failed to synchronize Process Group with Flow Registry : " + e.getMessage());
        versionControlFields.setSyncFailureExplanation(message);
        LOG.error("Failed to synchronize {} with Flow Registry because could not determine the most recent version of the Flow in the Flow Registry", this, e);
    }
}

Example 5 with VersionedProcessGroup

use of org.apache.nifi.registry.flow.VersionedProcessGroup in project nifi by apache.

the class StandardProcessGroup method verifyCanUpdate.

@Override
public void verifyCanUpdate(final VersionedFlowSnapshot updatedFlow, final boolean verifyConnectionRemoval, final boolean verifyNotDirty) {
    readLock.lock();
    try {
        final VersionControlInformation versionControlInfo = getVersionControlInformation();
        if (versionControlInfo != null) {
            if (!versionControlInfo.getFlowIdentifier().equals(updatedFlow.getSnapshotMetadata().getFlowIdentifier())) {
                throw new IllegalStateException(this + " is under version control but the given flow does not match the flow that this Process Group is synchronized with");
            }
            if (verifyNotDirty) {
                final VersionedFlowState flowState = versionControlInfo.getStatus().getState();
                final boolean modified = flowState == VersionedFlowState.LOCALLY_MODIFIED || flowState == VersionedFlowState.LOCALLY_MODIFIED_AND_STALE;
                final Set<FlowDifference> modifications = getModifications();
                if (modified) {
                    final String changes = modifications.stream().map(FlowDifference::toString).collect(Collectors.joining("\n"));
                    LOG.error("Cannot change the Version of the flow for {} because the Process Group has been modified ({} modifications) " + "since it was last synchronized with the Flow Registry. The following differences were found:\n{}", this, modifications.size(), changes);
                    throw new IllegalStateException("Cannot change the Version of the flow for " + this + " because the Process Group has been modified (" + modifications.size() + " modifications) since it was last synchronized with the Flow Registry. The Process Group must be" + " reverted to its original form before changing the version.");
                }
            }
            verifyNoDescendantsWithLocalModifications("be updated");
        }
        final VersionedProcessGroup flowContents = updatedFlow.getFlowContents();
        if (verifyConnectionRemoval) {
            // Determine which Connections have been removed.
            final Map<String, Connection> removedConnectionByVersionedId = new HashMap<>();
            // Populate the 'removedConnectionByVersionId' map with all Connections. We key off of the connection's VersionedComponentID
            // if it is populated. Otherwise, we key off of its actual ID. We do this because it allows us to then remove from this Map
            // any connection that does exist in the proposed flow. This results in us having a Map whose values are those Connections
            // that were removed. We can then check for any connections that have data in them. If any Connection is to be removed but
            // has data, then we should throw an IllegalStateException.
            findAllConnections().stream().forEach(conn -> removedConnectionByVersionedId.put(conn.getVersionedComponentId().orElse(conn.getIdentifier()), conn));
            final Set<String> proposedFlowConnectionIds = new HashSet<>();
            findAllConnectionIds(flowContents, proposedFlowConnectionIds);
            for (final String proposedConnectionId : proposedFlowConnectionIds) {
                removedConnectionByVersionedId.remove(proposedConnectionId);
            }
            // If any connection that was removed has data in it, throw an IllegalStateException
            for (final Connection connection : removedConnectionByVersionedId.values()) {
                final FlowFileQueue flowFileQueue = connection.getFlowFileQueue();
                if (!flowFileQueue.isEmpty()) {
                    throw new IllegalStateException(this + " cannot be updated to the proposed version of the flow because the " + "proposed version does not contain " + connection + " and the connection currently has data in the queue.");
                }
            }
        }
        // Determine which input ports were removed from this process group
        final Map<String, Port> removedInputPortsByVersionId = new HashMap<>();
        getInputPorts().stream().filter(port -> port.getVersionedComponentId().isPresent()).forEach(port -> removedInputPortsByVersionId.put(port.getVersionedComponentId().get(), port));
        flowContents.getInputPorts().stream().map(VersionedPort::getIdentifier).forEach(id -> removedInputPortsByVersionId.remove(id));
        // Ensure that there are no incoming connections for any Input Port that was removed.
        for (final Port inputPort : removedInputPortsByVersionId.values()) {
            final List<Connection> incomingConnections = inputPort.getIncomingConnections();
            if (!incomingConnections.isEmpty()) {
                throw new IllegalStateException(this + " cannot be updated to the proposed version of the flow because the proposed version does not contain the Input Port " + inputPort + " and the Input Port currently has an incoming connections");
            }
        }
        // Determine which output ports were removed from this process group
        final Map<String, Port> removedOutputPortsByVersionId = new HashMap<>();
        getOutputPorts().stream().filter(port -> port.getVersionedComponentId().isPresent()).forEach(port -> removedOutputPortsByVersionId.put(port.getVersionedComponentId().get(), port));
        flowContents.getOutputPorts().stream().map(VersionedPort::getIdentifier).forEach(id -> removedOutputPortsByVersionId.remove(id));
        // Ensure that there are no outgoing connections for any Output Port that was removed.
        for (final Port outputPort : removedOutputPortsByVersionId.values()) {
            final Set<Connection> outgoingConnections = outputPort.getConnections();
            if (!outgoingConnections.isEmpty()) {
                throw new IllegalStateException(this + " cannot be updated to the proposed version of the flow because the proposed version does not contain the Output Port " + outputPort + " and the Output Port currently has an outgoing connections");
            }
        }
        // Find any Process Groups that may have been deleted. If we find any Process Group that was deleted, and that Process Group
        // has Templates, then we fail because the Templates have to be removed first.
        final Map<String, VersionedProcessGroup> proposedProcessGroups = new HashMap<>();
        findAllProcessGroups(updatedFlow.getFlowContents(), proposedProcessGroups);
        for (final ProcessGroup childGroup : findAllProcessGroups()) {
            if (childGroup.getTemplates().isEmpty()) {
                continue;
            }
            final Optional<String> versionedIdOption = childGroup.getVersionedComponentId();
            if (!versionedIdOption.isPresent()) {
                continue;
            }
            final String versionedId = versionedIdOption.get();
            if (!proposedProcessGroups.containsKey(versionedId)) {
                // Process Group was removed.
                throw new IllegalStateException(this + " cannot be updated to the proposed version of the flow because the child " + childGroup + " that exists locally has one or more Templates, and the proposed flow does not contain these templates. " + "A Process Group cannot be deleted while it contains Templates. Please remove the Templates before attempting to change the version of the flow.");
            }
        }
        // Ensure that all Processors are instantiate-able.
        final Map<String, VersionedProcessor> proposedProcessors = new HashMap<>();
        findAllProcessors(updatedFlow.getFlowContents(), proposedProcessors);
        findAllProcessors().stream().filter(proc -> proc.getVersionedComponentId().isPresent()).forEach(proc -> proposedProcessors.remove(proc.getVersionedComponentId().get()));
        for (final VersionedProcessor processorToAdd : proposedProcessors.values()) {
            final BundleCoordinate coordinate = toCoordinate(processorToAdd.getBundle());
            try {
                flowController.createProcessor(processorToAdd.getType(), UUID.randomUUID().toString(), coordinate, false);
            } catch (Exception e) {
                throw new IllegalArgumentException("Unable to create Processor of type " + processorToAdd.getType(), e);
            }
        }
        // Ensure that all Controller Services are instantiate-able.
        final Map<String, VersionedControllerService> proposedServices = new HashMap<>();
        findAllControllerServices(updatedFlow.getFlowContents(), proposedServices);
        findAllControllerServices().stream().filter(service -> service.getVersionedComponentId().isPresent()).forEach(service -> proposedServices.remove(service.getVersionedComponentId().get()));
        for (final VersionedControllerService serviceToAdd : proposedServices.values()) {
            final BundleCoordinate coordinate = toCoordinate(serviceToAdd.getBundle());
            try {
                flowController.createControllerService(serviceToAdd.getType(), UUID.randomUUID().toString(), coordinate, Collections.emptySet(), false);
            } catch (Exception e) {
                throw new IllegalArgumentException("Unable to create Controller Service of type " + serviceToAdd.getType(), e);
            }
        }
        // Ensure that all Prioritizers are instantiate-able.
        final Map<String, VersionedConnection> proposedConnections = new HashMap<>();
        findAllConnections(updatedFlow.getFlowContents(), proposedConnections);
        findAllConnections().stream().filter(conn -> conn.getVersionedComponentId().isPresent()).forEach(conn -> proposedConnections.remove(conn.getVersionedComponentId().get()));
        for (final VersionedConnection connectionToAdd : proposedConnections.values()) {
            if (connectionToAdd.getPrioritizers() != null) {
                for (final String prioritizerType : connectionToAdd.getPrioritizers()) {
                    try {
                        flowController.createPrioritizer(prioritizerType);
                    } catch (Exception e) {
                        throw new IllegalArgumentException("Unable to create Prioritizer of type " + prioritizerType, e);
                    }
                }
            }
        }
    } finally {
        readLock.unlock();
    }
}