Examples with VersionedFlowSnapshot org.apache.nifi.registry.flow.VersionedFlowSnapshot used on opensource projects

Example 6 with VersionedFlowSnapshot

use of org.apache.nifi.registry.flow.VersionedFlowSnapshot in project nifi by apache.

the class ExportFlowVersion method doExecute.

@Override
public VersionedFlowSnapshotResult doExecute(final NiFiRegistryClient client, final Properties properties) throws ParseException, IOException, NiFiRegistryException {
    final String flowId = getRequiredArg(properties, CommandOption.FLOW_ID);
    final Integer version = getIntArg(properties, CommandOption.FLOW_VERSION);
    // determine the bucket for the provided flow id
    final String bucketId = getBucketId(client, flowId);
    // if no version was provided then export the latest, otherwise use specific version
    final VersionedFlowSnapshot versionedFlowSnapshot;
    if (version == null) {
        versionedFlowSnapshot = client.getFlowSnapshotClient().getLatest(bucketId, flowId);
    } else {
        versionedFlowSnapshot = client.getFlowSnapshotClient().get(bucketId, flowId, version);
    }
    versionedFlowSnapshot.setFlow(null);
    versionedFlowSnapshot.setBucket(null);
    versionedFlowSnapshot.getSnapshotMetadata().setBucketIdentifier(null);
    versionedFlowSnapshot.getSnapshotMetadata().setFlowIdentifier(null);
    versionedFlowSnapshot.getSnapshotMetadata().setLink(null);
    // currently export doesn't use the ResultWriter concept, it always writes JSON
    // destination will be a file if outputFile is specified, otherwise it will be the output stream of the CLI
    final String outputFile;
    if (properties.containsKey(CommandOption.OUTPUT_FILE.getLongName())) {
        outputFile = properties.getProperty(CommandOption.OUTPUT_FILE.getLongName());
    } else {
        outputFile = null;
    }
    return new VersionedFlowSnapshotResult(versionedFlowSnapshot, outputFile);
}

Example 7 with VersionedFlowSnapshot

use of org.apache.nifi.registry.flow.VersionedFlowSnapshot in project nifi by apache.

the class SyncFlowVersions method doExecute.

@Override
public StringResult doExecute(final NiFiRegistryClient client, final Properties properties) throws IOException, NiFiRegistryException, ParseException {
    final String srcPropsValue = getArg(properties, CommandOption.SRC_PROPS);
    final String srcFlowId = getRequiredArg(properties, CommandOption.SRC_FLOW_ID);
    final String destFlowId = getRequiredArg(properties, CommandOption.FLOW_ID);
    final NiFiRegistryClient srcClient = getSourceClient(client, srcPropsValue);
    final String srcBucketId = getBucketId(srcClient, srcFlowId);
    final String destBucketId = getBucketId(client, destFlowId);
    final List<Integer> srcVersions = getVersions(srcClient, srcBucketId, srcFlowId);
    final List<Integer> destVersions = getVersions(client, destBucketId, destFlowId);
    if (destVersions.size() > srcVersions.size()) {
        throw new NiFiRegistryException("Destination flow has more versions than source flow");
    }
    srcVersions.removeAll(destVersions);
    if (srcVersions.isEmpty()) {
        if (getContext().isInteractive()) {
            println();
            println("Source and destination already in sync");
        }
        return new OkResult(getContext().isInteractive());
    }
    // the REST API returns versions in decreasing order, but we want them in increasing order
    Collections.sort(srcVersions);
    for (final Integer srcVersion : srcVersions) {
        final VersionedFlowSnapshot srcFlowSnapshot = srcClient.getFlowSnapshotClient().get(srcBucketId, srcFlowId, srcVersion);
        srcFlowSnapshot.setFlow(null);
        srcFlowSnapshot.setBucket(null);
        final VersionedFlowSnapshotMetadata destMetadata = new VersionedFlowSnapshotMetadata();
        destMetadata.setBucketIdentifier(destBucketId);
        destMetadata.setFlowIdentifier(destFlowId);
        destMetadata.setVersion(srcVersion);
        destMetadata.setComments(srcFlowSnapshot.getSnapshotMetadata().getComments());
        srcFlowSnapshot.setSnapshotMetadata(destMetadata);
        client.getFlowSnapshotClient().create(srcFlowSnapshot);
        if (getContext().isInteractive()) {
            println();
            println("Synced version " + srcVersion);
        }
    }
    return new OkResult(getContext().isInteractive());
}

Example 8 with VersionedFlowSnapshot

use of org.apache.nifi.registry.flow.VersionedFlowSnapshot in project nifi by apache.

the class StandardProcessGroup method synchronizeWithFlowRegistry.

@Override
public void synchronizeWithFlowRegistry(final FlowRegistryClient flowRegistryClient) {
    final StandardVersionControlInformation vci = versionControlInfo.get();
    if (vci == null) {
        return;
    }
    final String registryId = vci.getRegistryIdentifier();
    final FlowRegistry flowRegistry = flowRegistryClient.getFlowRegistry(registryId);
    if (flowRegistry == null) {
        final String message = String.format("Unable to synchronize Process Group with Flow Registry because Process Group was placed under Version Control using Flow Registry " + "with identifier %s but cannot find any Flow Registry with this identifier", registryId);
        versionControlFields.setSyncFailureExplanation(message);
        LOG.error("Unable to synchronize {} with Flow Registry because Process Group was placed under Version Control using Flow Registry " + "with identifier {} but cannot find any Flow Registry with this identifier", this, registryId);
        return;
    }
    final VersionedProcessGroup snapshot = vci.getFlowSnapshot();
    if (snapshot == null) {
        // This allows us to know whether or not the flow has been modified since it was last synced with the Flow Registry.
        try {
            final VersionedFlowSnapshot registrySnapshot = flowRegistry.getFlowContents(vci.getBucketIdentifier(), vci.getFlowIdentifier(), vci.getVersion(), false);
            final VersionedProcessGroup registryFlow = registrySnapshot.getFlowContents();
            vci.setFlowSnapshot(registryFlow);
        } catch (final IOException | NiFiRegistryException e) {
            final String message = String.format("Failed to synchronize Process Group with Flow Registry because could not retrieve version %s of flow with identifier %s in bucket %s", vci.getVersion(), vci.getFlowIdentifier(), vci.getBucketIdentifier());
            versionControlFields.setSyncFailureExplanation(message);
            LOG.error("Failed to synchronize {} with Flow Registry because could not retrieve version {} of flow with identifier {} in bucket {}", this, vci.getVersion(), vci.getFlowIdentifier(), vci.getBucketIdentifier(), e);
            return;
        }
    }
    try {
        final VersionedFlow versionedFlow = flowRegistry.getVersionedFlow(vci.getBucketIdentifier(), vci.getFlowIdentifier());
        final int latestVersion = (int) versionedFlow.getVersionCount();
        vci.setBucketName(versionedFlow.getBucketName());
        vci.setFlowName(versionedFlow.getName());
        vci.setFlowDescription(versionedFlow.getDescription());
        vci.setRegistryName(flowRegistry.getName());
        if (latestVersion == vci.getVersion()) {
            LOG.debug("{} is currently at the most recent version ({}) of the flow that is under Version Control", this, latestVersion);
            versionControlFields.setStale(false);
        } else {
            LOG.info("{} is not the most recent version of the flow that is under Version Control; current version is {}; most recent version is {}", new Object[] { this, vci.getVersion(), latestVersion });
            versionControlFields.setStale(true);
        }
        versionControlFields.setSyncFailureExplanation(null);
    } catch (final IOException | NiFiRegistryException e) {
        final String message = String.format("Failed to synchronize Process Group with Flow Registry : " + e.getMessage());
        versionControlFields.setSyncFailureExplanation(message);
        LOG.error("Failed to synchronize {} with Flow Registry because could not determine the most recent version of the Flow in the Flow Registry", this, e);
    }
}

Example 9 with VersionedFlowSnapshot

use of org.apache.nifi.registry.flow.VersionedFlowSnapshot in project nifi by apache.

the class StandardProcessGroup method verifyCanUpdate.

@Override
public void verifyCanUpdate(final VersionedFlowSnapshot updatedFlow, final boolean verifyConnectionRemoval, final boolean verifyNotDirty) {
    readLock.lock();
    try {
        final VersionControlInformation versionControlInfo = getVersionControlInformation();
        if (versionControlInfo != null) {
            if (!versionControlInfo.getFlowIdentifier().equals(updatedFlow.getSnapshotMetadata().getFlowIdentifier())) {
                throw new IllegalStateException(this + " is under version control but the given flow does not match the flow that this Process Group is synchronized with");
            }
            if (verifyNotDirty) {
                final VersionedFlowState flowState = versionControlInfo.getStatus().getState();
                final boolean modified = flowState == VersionedFlowState.LOCALLY_MODIFIED || flowState == VersionedFlowState.LOCALLY_MODIFIED_AND_STALE;
                final Set<FlowDifference> modifications = getModifications();
                if (modified) {
                    final String changes = modifications.stream().map(FlowDifference::toString).collect(Collectors.joining("\n"));
                    LOG.error("Cannot change the Version of the flow for {} because the Process Group has been modified ({} modifications) " + "since it was last synchronized with the Flow Registry. The following differences were found:\n{}", this, modifications.size(), changes);
                    throw new IllegalStateException("Cannot change the Version of the flow for " + this + " because the Process Group has been modified (" + modifications.size() + " modifications) since it was last synchronized with the Flow Registry. The Process Group must be" + " reverted to its original form before changing the version.");
                }
            }
            verifyNoDescendantsWithLocalModifications("be updated");
        }
        final VersionedProcessGroup flowContents = updatedFlow.getFlowContents();
        if (verifyConnectionRemoval) {
            // Determine which Connections have been removed.
            final Map<String, Connection> removedConnectionByVersionedId = new HashMap<>();
            // Populate the 'removedConnectionByVersionId' map with all Connections. We key off of the connection's VersionedComponentID
            // if it is populated. Otherwise, we key off of its actual ID. We do this because it allows us to then remove from this Map
            // any connection that does exist in the proposed flow. This results in us having a Map whose values are those Connections
            // that were removed. We can then check for any connections that have data in them. If any Connection is to be removed but
            // has data, then we should throw an IllegalStateException.
            findAllConnections().stream().forEach(conn -> removedConnectionByVersionedId.put(conn.getVersionedComponentId().orElse(conn.getIdentifier()), conn));
            final Set<String> proposedFlowConnectionIds = new HashSet<>();
            findAllConnectionIds(flowContents, proposedFlowConnectionIds);
            for (final String proposedConnectionId : proposedFlowConnectionIds) {
                removedConnectionByVersionedId.remove(proposedConnectionId);
            }
            // If any connection that was removed has data in it, throw an IllegalStateException
            for (final Connection connection : removedConnectionByVersionedId.values()) {
                final FlowFileQueue flowFileQueue = connection.getFlowFileQueue();
                if (!flowFileQueue.isEmpty()) {
                    throw new IllegalStateException(this + " cannot be updated to the proposed version of the flow because the " + "proposed version does not contain " + connection + " and the connection currently has data in the queue.");
                }
            }
        }
        // Determine which input ports were removed from this process group
        final Map<String, Port> removedInputPortsByVersionId = new HashMap<>();
        getInputPorts().stream().filter(port -> port.getVersionedComponentId().isPresent()).forEach(port -> removedInputPortsByVersionId.put(port.getVersionedComponentId().get(), port));
        flowContents.getInputPorts().stream().map(VersionedPort::getIdentifier).forEach(id -> removedInputPortsByVersionId.remove(id));
        // Ensure that there are no incoming connections for any Input Port that was removed.
        for (final Port inputPort : removedInputPortsByVersionId.values()) {
            final List<Connection> incomingConnections = inputPort.getIncomingConnections();
            if (!incomingConnections.isEmpty()) {
                throw new IllegalStateException(this + " cannot be updated to the proposed version of the flow because the proposed version does not contain the Input Port " + inputPort + " and the Input Port currently has an incoming connections");
            }
        }
        // Determine which output ports were removed from this process group
        final Map<String, Port> removedOutputPortsByVersionId = new HashMap<>();
        getOutputPorts().stream().filter(port -> port.getVersionedComponentId().isPresent()).forEach(port -> removedOutputPortsByVersionId.put(port.getVersionedComponentId().get(), port));
        flowContents.getOutputPorts().stream().map(VersionedPort::getIdentifier).forEach(id -> removedOutputPortsByVersionId.remove(id));
        // Ensure that there are no outgoing connections for any Output Port that was removed.
        for (final Port outputPort : removedOutputPortsByVersionId.values()) {
            final Set<Connection> outgoingConnections = outputPort.getConnections();
            if (!outgoingConnections.isEmpty()) {
                throw new IllegalStateException(this + " cannot be updated to the proposed version of the flow because the proposed version does not contain the Output Port " + outputPort + " and the Output Port currently has an outgoing connections");
            }
        }
        // Find any Process Groups that may have been deleted. If we find any Process Group that was deleted, and that Process Group
        // has Templates, then we fail because the Templates have to be removed first.
        final Map<String, VersionedProcessGroup> proposedProcessGroups = new HashMap<>();
        findAllProcessGroups(updatedFlow.getFlowContents(), proposedProcessGroups);
        for (final ProcessGroup childGroup : findAllProcessGroups()) {
            if (childGroup.getTemplates().isEmpty()) {
                continue;
            }
            final Optional<String> versionedIdOption = childGroup.getVersionedComponentId();
            if (!versionedIdOption.isPresent()) {
                continue;
            }
            final String versionedId = versionedIdOption.get();
            if (!proposedProcessGroups.containsKey(versionedId)) {
                // Process Group was removed.
                throw new IllegalStateException(this + " cannot be updated to the proposed version of the flow because the child " + childGroup + " that exists locally has one or more Templates, and the proposed flow does not contain these templates. " + "A Process Group cannot be deleted while it contains Templates. Please remove the Templates before attempting to change the version of the flow.");
            }
        }
        // Ensure that all Processors are instantiate-able.
        final Map<String, VersionedProcessor> proposedProcessors = new HashMap<>();
        findAllProcessors(updatedFlow.getFlowContents(), proposedProcessors);
        findAllProcessors().stream().filter(proc -> proc.getVersionedComponentId().isPresent()).forEach(proc -> proposedProcessors.remove(proc.getVersionedComponentId().get()));
        for (final VersionedProcessor processorToAdd : proposedProcessors.values()) {
            final BundleCoordinate coordinate = toCoordinate(processorToAdd.getBundle());
            try {
                flowController.createProcessor(processorToAdd.getType(), UUID.randomUUID().toString(), coordinate, false);
            } catch (Exception e) {
                throw new IllegalArgumentException("Unable to create Processor of type " + processorToAdd.getType(), e);
            }
        }
        // Ensure that all Controller Services are instantiate-able.
        final Map<String, VersionedControllerService> proposedServices = new HashMap<>();
        findAllControllerServices(updatedFlow.getFlowContents(), proposedServices);
        findAllControllerServices().stream().filter(service -> service.getVersionedComponentId().isPresent()).forEach(service -> proposedServices.remove(service.getVersionedComponentId().get()));
        for (final VersionedControllerService serviceToAdd : proposedServices.values()) {
            final BundleCoordinate coordinate = toCoordinate(serviceToAdd.getBundle());
            try {
                flowController.createControllerService(serviceToAdd.getType(), UUID.randomUUID().toString(), coordinate, Collections.emptySet(), false);
            } catch (Exception e) {
                throw new IllegalArgumentException("Unable to create Controller Service of type " + serviceToAdd.getType(), e);
            }
        }
        // Ensure that all Prioritizers are instantiate-able.
        final Map<String, VersionedConnection> proposedConnections = new HashMap<>();
        findAllConnections(updatedFlow.getFlowContents(), proposedConnections);
        findAllConnections().stream().filter(conn -> conn.getVersionedComponentId().isPresent()).forEach(conn -> proposedConnections.remove(conn.getVersionedComponentId().get()));
        for (final VersionedConnection connectionToAdd : proposedConnections.values()) {
            if (connectionToAdd.getPrioritizers() != null) {
                for (final String prioritizerType : connectionToAdd.getPrioritizers()) {
                    try {
                        flowController.createPrioritizer(prioritizerType);
                    } catch (Exception e) {
                        throw new IllegalArgumentException("Unable to create Prioritizer of type " + prioritizerType, e);
                    }
                }
            }
        }
    } finally {
        readLock.unlock();
    }
}

Example 10 with VersionedFlowSnapshot

use of org.apache.nifi.registry.flow.VersionedFlowSnapshot in project nifi-registry by apache.

the class TestRegistryService method testCreateSnapshotNullMetadata.

@Test(expected = ConstraintViolationException.class)
public void testCreateSnapshotNullMetadata() {
    final VersionedFlowSnapshot snapshot = createSnapshot();
    snapshot.setSnapshotMetadata(null);
    registryService.createFlowSnapshot(snapshot);
}