Search in sources :

Example 1 with VersionedRemoteProcessGroup

use of org.apache.nifi.registry.flow.VersionedRemoteProcessGroup in project nifi by apache.

the class StandardProcessGroup method updateRemoteProcessGroup.

private void updateRemoteProcessGroup(final RemoteProcessGroup rpg, final VersionedRemoteProcessGroup proposed, final String componentIdSeed) {
    rpg.setComments(proposed.getComments());
    rpg.setCommunicationsTimeout(proposed.getCommunicationsTimeout());
    rpg.setInputPorts(proposed.getInputPorts() == null ? Collections.emptySet() : proposed.getInputPorts().stream().map(port -> createPortDescriptor(port, componentIdSeed, rpg.getIdentifier())).collect(Collectors.toSet()), false);
    rpg.setName(proposed.getName());
    rpg.setNetworkInterface(proposed.getLocalNetworkInterface());
    rpg.setOutputPorts(proposed.getOutputPorts() == null ? Collections.emptySet() : proposed.getOutputPorts().stream().map(port -> createPortDescriptor(port, componentIdSeed, rpg.getIdentifier())).collect(Collectors.toSet()), false);
    rpg.setPosition(new Position(proposed.getPosition().getX(), proposed.getPosition().getY()));
    rpg.setProxyHost(proposed.getProxyHost());
    rpg.setProxyPort(proposed.getProxyPort());
    rpg.setProxyUser(proposed.getProxyUser());
    rpg.setTransportProtocol(SiteToSiteTransportProtocol.valueOf(proposed.getTransportProtocol()));
    rpg.setYieldDuration(proposed.getYieldDuration());
}
Also used : OnRemoved(org.apache.nifi.annotation.lifecycle.OnRemoved) ConfigurationContext(org.apache.nifi.controller.ConfigurationContext) Size(org.apache.nifi.connectable.Size) FlowComparison(org.apache.nifi.registry.flow.diff.FlowComparison) StringUtils(org.apache.commons.lang3.StringUtils) ReflectionUtils(org.apache.nifi.util.ReflectionUtils) PropertyDescriptor(org.apache.nifi.components.PropertyDescriptor) SecureRandom(java.security.SecureRandom) NiFiRegistryException(org.apache.nifi.registry.client.NiFiRegistryException) ComponentType(org.apache.nifi.registry.flow.ComponentType) VersionedProcessGroup(org.apache.nifi.registry.flow.VersionedProcessGroup) SnippetUtils(org.apache.nifi.util.SnippetUtils) Map(java.util.Map) HashCodeBuilder(org.apache.commons.lang3.builder.HashCodeBuilder) RootGroupPort(org.apache.nifi.remote.RootGroupPort) Connectable(org.apache.nifi.connectable.Connectable) Connection(org.apache.nifi.connectable.Connection) Bundle(org.apache.nifi.registry.flow.Bundle) FlowFilePrioritizer(org.apache.nifi.flowfile.FlowFilePrioritizer) FlowDifferenceFilters(org.apache.nifi.util.FlowDifferenceFilters) VersionedFlowStatus(org.apache.nifi.registry.flow.VersionedFlowStatus) Set(java.util.Set) VersionedFlowCoordinates(org.apache.nifi.registry.flow.VersionedFlowCoordinates) VersionedRemoteGroupPort(org.apache.nifi.registry.flow.VersionedRemoteGroupPort) FlowController(org.apache.nifi.controller.FlowController) StandardCharsets(java.nio.charset.StandardCharsets) StateManagerProvider(org.apache.nifi.components.state.StateManagerProvider) Position(org.apache.nifi.connectable.Position) ScheduledState(org.apache.nifi.controller.ScheduledState) ControllerService(org.apache.nifi.controller.ControllerService) ExtensionManager(org.apache.nifi.nar.ExtensionManager) StandardVersionControlInformation(org.apache.nifi.registry.flow.StandardVersionControlInformation) Resource(org.apache.nifi.authorization.Resource) FlowComparator(org.apache.nifi.registry.flow.diff.FlowComparator) StaticDifferenceDescriptor(org.apache.nifi.registry.flow.diff.StaticDifferenceDescriptor) StandardComparableDataFlow(org.apache.nifi.registry.flow.diff.StandardComparableDataFlow) SiteToSiteTransportProtocol(org.apache.nifi.remote.protocol.SiteToSiteTransportProtocol) ReentrantReadWriteLock(java.util.concurrent.locks.ReentrantReadWriteLock) ArrayList(java.util.ArrayList) Relationship(org.apache.nifi.processor.Relationship) ControllerServiceReference(org.apache.nifi.controller.service.ControllerServiceReference) ControllerServiceProvider(org.apache.nifi.controller.service.ControllerServiceProvider) VersionedLabel(org.apache.nifi.registry.flow.VersionedLabel) LinkedHashSet(java.util.LinkedHashSet) VersionedFlowState(org.apache.nifi.registry.flow.VersionedFlowState) EvolvingDifferenceDescriptor(org.apache.nifi.registry.flow.diff.EvolvingDifferenceDescriptor) ConfiguredComponent(org.apache.nifi.controller.ConfiguredComponent) Positionable(org.apache.nifi.connectable.Positionable) ExecutionNode(org.apache.nifi.scheduling.ExecutionNode) IOException(java.io.IOException) VersionedFlowSnapshot(org.apache.nifi.registry.flow.VersionedFlowSnapshot) NiFiRegistryFlowMapper(org.apache.nifi.registry.flow.mapping.NiFiRegistryFlowMapper) Lock(java.util.concurrent.locks.Lock) NiFiProperties(org.apache.nifi.util.NiFiProperties) VariableImpact(org.apache.nifi.attribute.expression.language.VariableImpact) FlowFileQueue(org.apache.nifi.controller.queue.FlowFileQueue) ProcessorInstantiationException(org.apache.nifi.controller.exception.ProcessorInstantiationException) BundleCoordinate(org.apache.nifi.bundle.BundleCoordinate) URL(java.net.URL) ConnectableType(org.apache.nifi.connectable.ConnectableType) ConnectableComponent(org.apache.nifi.registry.flow.ConnectableComponent) VariableDescriptor(org.apache.nifi.registry.VariableDescriptor) LoggerFactory(org.slf4j.LoggerFactory) Port(org.apache.nifi.connectable.Port) StandardFlowComparator(org.apache.nifi.registry.flow.diff.StandardFlowComparator) Query(org.apache.nifi.attribute.expression.language.Query) ResourceType(org.apache.nifi.authorization.resource.ResourceType) TemplateDTO(org.apache.nifi.web.api.dto.TemplateDTO) SchedulingStrategy(org.apache.nifi.scheduling.SchedulingStrategy) VersionedPort(org.apache.nifi.registry.flow.VersionedPort) VersionedRemoteProcessGroup(org.apache.nifi.registry.flow.VersionedRemoteProcessGroup) StandardProcessScheduler(org.apache.nifi.controller.scheduling.StandardProcessScheduler) VersionedComponent(org.apache.nifi.registry.flow.VersionedComponent) DifferenceType(org.apache.nifi.registry.flow.diff.DifferenceType) VersionedConnection(org.apache.nifi.registry.flow.VersionedConnection) Template(org.apache.nifi.controller.Template) Label(org.apache.nifi.controller.label.Label) FlowRegistryClient(org.apache.nifi.registry.flow.FlowRegistryClient) OnShutdown(org.apache.nifi.annotation.lifecycle.OnShutdown) MutableVariableRegistry(org.apache.nifi.registry.variable.MutableVariableRegistry) Authorizable(org.apache.nifi.authorization.resource.Authorizable) UUID(java.util.UUID) ComponentLifeCycleException(org.apache.nifi.controller.exception.ComponentLifeCycleException) Snippet(org.apache.nifi.controller.Snippet) Collectors(java.util.stream.Collectors) ResourceFactory(org.apache.nifi.authorization.resource.ResourceFactory) Objects(java.util.Objects) List(java.util.List) BatchSize(org.apache.nifi.registry.flow.BatchSize) VersionedFunnel(org.apache.nifi.registry.flow.VersionedFunnel) ToStringBuilder(org.apache.commons.lang3.builder.ToStringBuilder) VersionControlInformation(org.apache.nifi.registry.flow.VersionControlInformation) Optional(java.util.Optional) LocalPort(org.apache.nifi.connectable.LocalPort) StandardProcessContext(org.apache.nifi.processor.StandardProcessContext) ProcessorNode(org.apache.nifi.controller.ProcessorNode) Revision(org.apache.nifi.web.Revision) Funnel(org.apache.nifi.connectable.Funnel) ControllerServiceNode(org.apache.nifi.controller.service.ControllerServiceNode) ToStringStyle(org.apache.commons.lang3.builder.ToStringStyle) HashMap(java.util.HashMap) CompletableFuture(java.util.concurrent.CompletableFuture) AtomicReference(java.util.concurrent.atomic.AtomicReference) Function(java.util.function.Function) FlowRegistry(org.apache.nifi.registry.flow.FlowRegistry) HashSet(java.util.HashSet) StringEncryptor(org.apache.nifi.encrypt.StringEncryptor) ComparableDataFlow(org.apache.nifi.registry.flow.diff.ComparableDataFlow) Objects.requireNonNull(java.util.Objects.requireNonNull) StandardConfigurationContext(org.apache.nifi.controller.service.StandardConfigurationContext) NarCloseable(org.apache.nifi.nar.NarCloseable) LogLevel(org.apache.nifi.logging.LogLevel) VersionedProcessor(org.apache.nifi.registry.flow.VersionedProcessor) Logger(org.slf4j.Logger) StateManager(org.apache.nifi.components.state.StateManager) RemoteGroupPort(org.apache.nifi.remote.RemoteGroupPort) StandardRemoteProcessGroupPortDescriptor(org.apache.nifi.remote.StandardRemoteProcessGroupPortDescriptor) VersionedFlow(org.apache.nifi.registry.flow.VersionedFlow) VersionedControllerService(org.apache.nifi.registry.flow.VersionedControllerService) TimeUnit(java.util.concurrent.TimeUnit) ComponentVariableRegistry(org.apache.nifi.registry.ComponentVariableRegistry) FlowDifference(org.apache.nifi.registry.flow.diff.FlowDifference) VersionedPropertyDescriptor(org.apache.nifi.registry.flow.VersionedPropertyDescriptor) Collections(java.util.Collections) LogRepositoryFactory(org.apache.nifi.logging.LogRepositoryFactory) Position(org.apache.nifi.connectable.Position)

Example 2 with VersionedRemoteProcessGroup

use of org.apache.nifi.registry.flow.VersionedRemoteProcessGroup in project nifi by apache.

the class StandardProcessGroup method updateProcessGroup.

private void updateProcessGroup(final ProcessGroup group, final VersionedProcessGroup proposed, final String componentIdSeed, final Set<String> updatedVersionedComponentIds, final boolean updatePosition, final boolean updateName, final boolean updateDescendantVersionedGroups, final Set<String> variablesToSkip) throws ProcessorInstantiationException {
    group.setComments(proposed.getComments());
    if (updateName) {
        group.setName(proposed.getName());
    }
    if (updatePosition && proposed.getPosition() != null) {
        group.setPosition(new Position(proposed.getPosition().getX(), proposed.getPosition().getY()));
    }
    // Determine which variables have been added/removed and add/remove them from this group's variable registry.
    // We don't worry about if a variable value has changed, because variables are designed to be 'environment specific.'
    // As a result, once imported, we won't update variables to match the remote flow, but we will add any missing variables
    // and remove any variables that are no longer part of the remote flow.
    final Set<String> existingVariableNames = group.getVariableRegistry().getVariableMap().keySet().stream().map(VariableDescriptor::getName).collect(Collectors.toSet());
    final Map<String, String> updatedVariableMap = new HashMap<>();
    // If any new variables exist in the proposed flow, add those to the variable registry.
    for (final Map.Entry<String, String> entry : proposed.getVariables().entrySet()) {
        if (!existingVariableNames.contains(entry.getKey()) && !variablesToSkip.contains(entry.getKey())) {
            updatedVariableMap.put(entry.getKey(), entry.getValue());
        }
    }
    group.setVariables(updatedVariableMap);
    final VersionedFlowCoordinates remoteCoordinates = proposed.getVersionedFlowCoordinates();
    if (remoteCoordinates == null) {
        group.disconnectVersionControl(false);
    } else {
        final String registryId = flowController.getFlowRegistryClient().getFlowRegistryId(remoteCoordinates.getRegistryUrl());
        final String bucketId = remoteCoordinates.getBucketId();
        final String flowId = remoteCoordinates.getFlowId();
        final int version = remoteCoordinates.getVersion();
        final FlowRegistry flowRegistry = flowController.getFlowRegistryClient().getFlowRegistry(registryId);
        final String registryName = flowRegistry == null ? registryId : flowRegistry.getName();
        final VersionedFlowState flowState = remoteCoordinates.getLatest() ? VersionedFlowState.UP_TO_DATE : VersionedFlowState.STALE;
        final VersionControlInformation vci = new StandardVersionControlInformation.Builder().registryId(registryId).registryName(registryName).bucketId(bucketId).bucketName(bucketId).flowId(flowId).flowName(flowId).version(version).flowSnapshot(proposed).status(new StandardVersionedFlowStatus(flowState, flowState.getDescription())).build();
        group.setVersionControlInformation(vci, Collections.emptyMap());
    }
    // Controller Services
    // Controller Services have to be handled a bit differently than other components. This is because Processors and Controller
    // Services may reference other Controller Services. Since we may be adding Service A, which depends on Service B, before adding
    // Service B, we need to ensure that we create all Controller Services first and then call updateControllerService for each
    // Controller Service. This way, we ensure that all services have been created before setting the properties. This allows us to
    // properly obtain the correct mapping of Controller Service VersionedComponentID to Controller Service instance id.
    final Map<String, ControllerServiceNode> servicesByVersionedId = group.getControllerServices(false).stream().collect(Collectors.toMap(component -> component.getVersionedComponentId().orElse(component.getIdentifier()), Function.identity()));
    final Set<String> controllerServicesRemoved = new HashSet<>(servicesByVersionedId.keySet());
    final Map<ControllerServiceNode, VersionedControllerService> services = new HashMap<>();
    // Add any Controller Service that does not yet exist.
    for (final VersionedControllerService proposedService : proposed.getControllerServices()) {
        ControllerServiceNode service = servicesByVersionedId.get(proposedService.getIdentifier());
        if (service == null) {
            service = addControllerService(group, proposedService, componentIdSeed);
            LOG.info("Added {} to {}", service, this);
        }
        services.put(service, proposedService);
    }
    // Update all of the Controller Services to match the VersionedControllerService
    for (final Map.Entry<ControllerServiceNode, VersionedControllerService> entry : services.entrySet()) {
        final ControllerServiceNode service = entry.getKey();
        final VersionedControllerService proposedService = entry.getValue();
        if (updatedVersionedComponentIds.contains(proposedService.getIdentifier())) {
            updateControllerService(service, proposedService);
            LOG.info("Updated {}", service);
        }
        controllerServicesRemoved.remove(proposedService.getIdentifier());
    }
    // Child groups
    final Map<String, ProcessGroup> childGroupsByVersionedId = group.getProcessGroups().stream().collect(Collectors.toMap(component -> component.getVersionedComponentId().orElse(component.getIdentifier()), Function.identity()));
    final Set<String> childGroupsRemoved = new HashSet<>(childGroupsByVersionedId.keySet());
    for (final VersionedProcessGroup proposedChildGroup : proposed.getProcessGroups()) {
        final ProcessGroup childGroup = childGroupsByVersionedId.get(proposedChildGroup.getIdentifier());
        final VersionedFlowCoordinates childCoordinates = proposedChildGroup.getVersionedFlowCoordinates();
        if (childGroup == null) {
            final ProcessGroup added = addProcessGroup(group, proposedChildGroup, componentIdSeed, variablesToSkip);
            flowController.onProcessGroupAdded(added);
            added.findAllRemoteProcessGroups().stream().forEach(RemoteProcessGroup::initialize);
            LOG.info("Added {} to {}", added, this);
        } else if (childCoordinates == null || updateDescendantVersionedGroups) {
            updateProcessGroup(childGroup, proposedChildGroup, componentIdSeed, updatedVersionedComponentIds, true, true, updateDescendantVersionedGroups, variablesToSkip);
            LOG.info("Updated {}", childGroup);
        }
        childGroupsRemoved.remove(proposedChildGroup.getIdentifier());
    }
    // Funnels
    final Map<String, Funnel> funnelsByVersionedId = group.getFunnels().stream().collect(Collectors.toMap(component -> component.getVersionedComponentId().orElse(component.getIdentifier()), Function.identity()));
    final Set<String> funnelsRemoved = new HashSet<>(funnelsByVersionedId.keySet());
    for (final VersionedFunnel proposedFunnel : proposed.getFunnels()) {
        final Funnel funnel = funnelsByVersionedId.get(proposedFunnel.getIdentifier());
        if (funnel == null) {
            final Funnel added = addFunnel(group, proposedFunnel, componentIdSeed);
            flowController.onFunnelAdded(added);
            LOG.info("Added {} to {}", added, this);
        } else if (updatedVersionedComponentIds.contains(proposedFunnel.getIdentifier())) {
            updateFunnel(funnel, proposedFunnel);
            LOG.info("Updated {}", funnel);
        } else {
            funnel.setPosition(new Position(proposedFunnel.getPosition().getX(), proposedFunnel.getPosition().getY()));
        }
        funnelsRemoved.remove(proposedFunnel.getIdentifier());
    }
    // Input Ports
    final Map<String, Port> inputPortsByVersionedId = group.getInputPorts().stream().collect(Collectors.toMap(component -> component.getVersionedComponentId().orElse(component.getIdentifier()), Function.identity()));
    final Set<String> inputPortsRemoved = new HashSet<>(inputPortsByVersionedId.keySet());
    for (final VersionedPort proposedPort : proposed.getInputPorts()) {
        final Port port = inputPortsByVersionedId.get(proposedPort.getIdentifier());
        if (port == null) {
            final Port added = addInputPort(group, proposedPort, componentIdSeed);
            flowController.onInputPortAdded(added);
            LOG.info("Added {} to {}", added, this);
        } else if (updatedVersionedComponentIds.contains(proposedPort.getIdentifier())) {
            updatePort(port, proposedPort);
            LOG.info("Updated {}", port);
        } else {
            port.setPosition(new Position(proposedPort.getPosition().getX(), proposedPort.getPosition().getY()));
        }
        inputPortsRemoved.remove(proposedPort.getIdentifier());
    }
    // Output Ports
    final Map<String, Port> outputPortsByVersionedId = group.getOutputPorts().stream().collect(Collectors.toMap(component -> component.getVersionedComponentId().orElse(component.getIdentifier()), Function.identity()));
    final Set<String> outputPortsRemoved = new HashSet<>(outputPortsByVersionedId.keySet());
    for (final VersionedPort proposedPort : proposed.getOutputPorts()) {
        final Port port = outputPortsByVersionedId.get(proposedPort.getIdentifier());
        if (port == null) {
            final Port added = addOutputPort(group, proposedPort, componentIdSeed);
            flowController.onOutputPortAdded(added);
            LOG.info("Added {} to {}", added, this);
        } else if (updatedVersionedComponentIds.contains(proposedPort.getIdentifier())) {
            updatePort(port, proposedPort);
            LOG.info("Updated {}", port);
        } else {
            port.setPosition(new Position(proposedPort.getPosition().getX(), proposedPort.getPosition().getY()));
        }
        outputPortsRemoved.remove(proposedPort.getIdentifier());
    }
    // Labels
    final Map<String, Label> labelsByVersionedId = group.getLabels().stream().collect(Collectors.toMap(component -> component.getVersionedComponentId().orElse(component.getIdentifier()), Function.identity()));
    final Set<String> labelsRemoved = new HashSet<>(labelsByVersionedId.keySet());
    for (final VersionedLabel proposedLabel : proposed.getLabels()) {
        final Label label = labelsByVersionedId.get(proposedLabel.getIdentifier());
        if (label == null) {
            final Label added = addLabel(group, proposedLabel, componentIdSeed);
            LOG.info("Added {} to {}", added, this);
        } else if (updatedVersionedComponentIds.contains(proposedLabel.getIdentifier())) {
            updateLabel(label, proposedLabel);
            LOG.info("Updated {}", label);
        } else {
            label.setPosition(new Position(proposedLabel.getPosition().getX(), proposedLabel.getPosition().getY()));
        }
        labelsRemoved.remove(proposedLabel.getIdentifier());
    }
    // Processors
    final Map<String, ProcessorNode> processorsByVersionedId = group.getProcessors().stream().collect(Collectors.toMap(component -> component.getVersionedComponentId().orElse(component.getIdentifier()), Function.identity()));
    final Set<String> processorsRemoved = new HashSet<>(processorsByVersionedId.keySet());
    final Map<ProcessorNode, Set<Relationship>> autoTerminatedRelationships = new HashMap<>();
    for (final VersionedProcessor proposedProcessor : proposed.getProcessors()) {
        final ProcessorNode processor = processorsByVersionedId.get(proposedProcessor.getIdentifier());
        if (processor == null) {
            final ProcessorNode added = addProcessor(group, proposedProcessor, componentIdSeed);
            flowController.onProcessorAdded(added);
            final Set<Relationship> proposedAutoTerminated = proposedProcessor.getAutoTerminatedRelationships() == null ? Collections.emptySet() : proposedProcessor.getAutoTerminatedRelationships().stream().map(relName -> added.getRelationship(relName)).collect(Collectors.toSet());
            autoTerminatedRelationships.put(added, proposedAutoTerminated);
            LOG.info("Added {} to {}", added, this);
        } else if (updatedVersionedComponentIds.contains(proposedProcessor.getIdentifier())) {
            updateProcessor(processor, proposedProcessor);
            final Set<Relationship> proposedAutoTerminated = proposedProcessor.getAutoTerminatedRelationships() == null ? Collections.emptySet() : proposedProcessor.getAutoTerminatedRelationships().stream().map(relName -> processor.getRelationship(relName)).collect(Collectors.toSet());
            if (!processor.getAutoTerminatedRelationships().equals(proposedAutoTerminated)) {
                autoTerminatedRelationships.put(processor, proposedAutoTerminated);
            }
            LOG.info("Updated {}", processor);
        } else {
            processor.setPosition(new Position(proposedProcessor.getPosition().getX(), proposedProcessor.getPosition().getY()));
        }
        processorsRemoved.remove(proposedProcessor.getIdentifier());
    }
    // Remote Groups
    final Map<String, RemoteProcessGroup> rpgsByVersionedId = group.getRemoteProcessGroups().stream().collect(Collectors.toMap(component -> component.getVersionedComponentId().orElse(component.getIdentifier()), Function.identity()));
    final Set<String> rpgsRemoved = new HashSet<>(rpgsByVersionedId.keySet());
    for (final VersionedRemoteProcessGroup proposedRpg : proposed.getRemoteProcessGroups()) {
        final RemoteProcessGroup rpg = rpgsByVersionedId.get(proposedRpg.getIdentifier());
        if (rpg == null) {
            final RemoteProcessGroup added = addRemoteProcessGroup(group, proposedRpg, componentIdSeed);
            LOG.info("Added {} to {}", added, this);
        } else if (updatedVersionedComponentIds.contains(proposedRpg.getIdentifier())) {
            updateRemoteProcessGroup(rpg, proposedRpg, componentIdSeed);
            LOG.info("Updated {}", rpg);
        } else {
            rpg.setPosition(new Position(proposedRpg.getPosition().getX(), proposedRpg.getPosition().getY()));
        }
        rpgsRemoved.remove(proposedRpg.getIdentifier());
    }
    // Connections
    final Map<String, Connection> connectionsByVersionedId = group.getConnections().stream().collect(Collectors.toMap(component -> component.getVersionedComponentId().orElse(component.getIdentifier()), Function.identity()));
    final Set<String> connectionsRemoved = new HashSet<>(connectionsByVersionedId.keySet());
    for (final VersionedConnection proposedConnection : proposed.getConnections()) {
        final Connection connection = connectionsByVersionedId.get(proposedConnection.getIdentifier());
        if (connection == null) {
            final Connection added = addConnection(group, proposedConnection, componentIdSeed);
            flowController.onConnectionAdded(added);
            LOG.info("Added {} to {}", added, this);
        } else if (isUpdateable(connection)) {
            // If the connection needs to be updated, then the source and destination will already have
            // been stopped (else, the validation above would fail). So if the source or the destination is running,
            // then we know that we don't need to update the connection.
            updateConnection(connection, proposedConnection);
            LOG.info("Updated {}", connection);
        }
        connectionsRemoved.remove(proposedConnection.getIdentifier());
    }
    // to remove a component if it has a connection going to it!
    for (final String removedVersionedId : connectionsRemoved) {
        final Connection connection = connectionsByVersionedId.get(removedVersionedId);
        LOG.info("Removing {} from {}", connection, group);
        group.removeConnection(connection);
        flowController.onConnectionRemoved(connection);
    }
    // Once the appropriate connections have been removed, we may now update Processors' auto-terminated relationships.
    // We cannot do this above, in the 'updateProcessor' call because if a connection is removed and changed to auto-terminated,
    // then updating this in the updateProcessor call above would attempt to set the Relationship to being auto-terminated while a
    // Connection for that relationship exists. This will throw an Exception.
    autoTerminatedRelationships.forEach((proc, rels) -> proc.setAutoTerminatedRelationships(rels));
    // Remove all controller services no longer in use
    for (final String removedVersionedId : controllerServicesRemoved) {
        final ControllerServiceNode service = servicesByVersionedId.get(removedVersionedId);
        LOG.info("Removing {} from {}", service, group);
        // Must remove Controller Service through Flow Controller in order to remove from cache
        flowController.removeControllerService(service);
    }
    for (final String removedVersionedId : funnelsRemoved) {
        final Funnel funnel = funnelsByVersionedId.get(removedVersionedId);
        LOG.info("Removing {} from {}", funnel, group);
        group.removeFunnel(funnel);
    }
    for (final String removedVersionedId : inputPortsRemoved) {
        final Port port = inputPortsByVersionedId.get(removedVersionedId);
        LOG.info("Removing {} from {}", port, group);
        group.removeInputPort(port);
    }
    for (final String removedVersionedId : outputPortsRemoved) {
        final Port port = outputPortsByVersionedId.get(removedVersionedId);
        LOG.info("Removing {} from {}", port, group);
        group.removeOutputPort(port);
    }
    for (final String removedVersionedId : labelsRemoved) {
        final Label label = labelsByVersionedId.get(removedVersionedId);
        LOG.info("Removing {} from {}", label, group);
        group.removeLabel(label);
    }
    for (final String removedVersionedId : processorsRemoved) {
        final ProcessorNode processor = processorsByVersionedId.get(removedVersionedId);
        LOG.info("Removing {} from {}", processor, group);
        group.removeProcessor(processor);
    }
    for (final String removedVersionedId : rpgsRemoved) {
        final RemoteProcessGroup rpg = rpgsByVersionedId.get(removedVersionedId);
        LOG.info("Removing {} from {}", rpg, group);
        group.removeRemoteProcessGroup(rpg);
    }
    for (final String removedVersionedId : childGroupsRemoved) {
        final ProcessGroup childGroup = childGroupsByVersionedId.get(removedVersionedId);
        LOG.info("Removing {} from {}", childGroup, group);
        group.removeProcessGroup(childGroup);
    }
}
Also used : OnRemoved(org.apache.nifi.annotation.lifecycle.OnRemoved) ConfigurationContext(org.apache.nifi.controller.ConfigurationContext) Size(org.apache.nifi.connectable.Size) FlowComparison(org.apache.nifi.registry.flow.diff.FlowComparison) StringUtils(org.apache.commons.lang3.StringUtils) ReflectionUtils(org.apache.nifi.util.ReflectionUtils) PropertyDescriptor(org.apache.nifi.components.PropertyDescriptor) SecureRandom(java.security.SecureRandom) NiFiRegistryException(org.apache.nifi.registry.client.NiFiRegistryException) ComponentType(org.apache.nifi.registry.flow.ComponentType) VersionedProcessGroup(org.apache.nifi.registry.flow.VersionedProcessGroup) SnippetUtils(org.apache.nifi.util.SnippetUtils) Map(java.util.Map) HashCodeBuilder(org.apache.commons.lang3.builder.HashCodeBuilder) RootGroupPort(org.apache.nifi.remote.RootGroupPort) Connectable(org.apache.nifi.connectable.Connectable) Connection(org.apache.nifi.connectable.Connection) Bundle(org.apache.nifi.registry.flow.Bundle) FlowFilePrioritizer(org.apache.nifi.flowfile.FlowFilePrioritizer) FlowDifferenceFilters(org.apache.nifi.util.FlowDifferenceFilters) VersionedFlowStatus(org.apache.nifi.registry.flow.VersionedFlowStatus) Set(java.util.Set) VersionedFlowCoordinates(org.apache.nifi.registry.flow.VersionedFlowCoordinates) VersionedRemoteGroupPort(org.apache.nifi.registry.flow.VersionedRemoteGroupPort) FlowController(org.apache.nifi.controller.FlowController) StandardCharsets(java.nio.charset.StandardCharsets) StateManagerProvider(org.apache.nifi.components.state.StateManagerProvider) Position(org.apache.nifi.connectable.Position) ScheduledState(org.apache.nifi.controller.ScheduledState) ControllerService(org.apache.nifi.controller.ControllerService) ExtensionManager(org.apache.nifi.nar.ExtensionManager) StandardVersionControlInformation(org.apache.nifi.registry.flow.StandardVersionControlInformation) Resource(org.apache.nifi.authorization.Resource) FlowComparator(org.apache.nifi.registry.flow.diff.FlowComparator) StaticDifferenceDescriptor(org.apache.nifi.registry.flow.diff.StaticDifferenceDescriptor) StandardComparableDataFlow(org.apache.nifi.registry.flow.diff.StandardComparableDataFlow) SiteToSiteTransportProtocol(org.apache.nifi.remote.protocol.SiteToSiteTransportProtocol) ReentrantReadWriteLock(java.util.concurrent.locks.ReentrantReadWriteLock) ArrayList(java.util.ArrayList) Relationship(org.apache.nifi.processor.Relationship) ControllerServiceReference(org.apache.nifi.controller.service.ControllerServiceReference) ControllerServiceProvider(org.apache.nifi.controller.service.ControllerServiceProvider) VersionedLabel(org.apache.nifi.registry.flow.VersionedLabel) LinkedHashSet(java.util.LinkedHashSet) VersionedFlowState(org.apache.nifi.registry.flow.VersionedFlowState) EvolvingDifferenceDescriptor(org.apache.nifi.registry.flow.diff.EvolvingDifferenceDescriptor) ConfiguredComponent(org.apache.nifi.controller.ConfiguredComponent) Positionable(org.apache.nifi.connectable.Positionable) ExecutionNode(org.apache.nifi.scheduling.ExecutionNode) IOException(java.io.IOException) VersionedFlowSnapshot(org.apache.nifi.registry.flow.VersionedFlowSnapshot) NiFiRegistryFlowMapper(org.apache.nifi.registry.flow.mapping.NiFiRegistryFlowMapper) Lock(java.util.concurrent.locks.Lock) NiFiProperties(org.apache.nifi.util.NiFiProperties) VariableImpact(org.apache.nifi.attribute.expression.language.VariableImpact) FlowFileQueue(org.apache.nifi.controller.queue.FlowFileQueue) ProcessorInstantiationException(org.apache.nifi.controller.exception.ProcessorInstantiationException) BundleCoordinate(org.apache.nifi.bundle.BundleCoordinate) URL(java.net.URL) ConnectableType(org.apache.nifi.connectable.ConnectableType) ConnectableComponent(org.apache.nifi.registry.flow.ConnectableComponent) VariableDescriptor(org.apache.nifi.registry.VariableDescriptor) LoggerFactory(org.slf4j.LoggerFactory) Port(org.apache.nifi.connectable.Port) StandardFlowComparator(org.apache.nifi.registry.flow.diff.StandardFlowComparator) Query(org.apache.nifi.attribute.expression.language.Query) ResourceType(org.apache.nifi.authorization.resource.ResourceType) TemplateDTO(org.apache.nifi.web.api.dto.TemplateDTO) SchedulingStrategy(org.apache.nifi.scheduling.SchedulingStrategy) VersionedPort(org.apache.nifi.registry.flow.VersionedPort) VersionedRemoteProcessGroup(org.apache.nifi.registry.flow.VersionedRemoteProcessGroup) StandardProcessScheduler(org.apache.nifi.controller.scheduling.StandardProcessScheduler) VersionedComponent(org.apache.nifi.registry.flow.VersionedComponent) DifferenceType(org.apache.nifi.registry.flow.diff.DifferenceType) VersionedConnection(org.apache.nifi.registry.flow.VersionedConnection) Template(org.apache.nifi.controller.Template) Label(org.apache.nifi.controller.label.Label) FlowRegistryClient(org.apache.nifi.registry.flow.FlowRegistryClient) OnShutdown(org.apache.nifi.annotation.lifecycle.OnShutdown) MutableVariableRegistry(org.apache.nifi.registry.variable.MutableVariableRegistry) Authorizable(org.apache.nifi.authorization.resource.Authorizable) UUID(java.util.UUID) ComponentLifeCycleException(org.apache.nifi.controller.exception.ComponentLifeCycleException) Snippet(org.apache.nifi.controller.Snippet) Collectors(java.util.stream.Collectors) ResourceFactory(org.apache.nifi.authorization.resource.ResourceFactory) Objects(java.util.Objects) List(java.util.List) BatchSize(org.apache.nifi.registry.flow.BatchSize) VersionedFunnel(org.apache.nifi.registry.flow.VersionedFunnel) ToStringBuilder(org.apache.commons.lang3.builder.ToStringBuilder) VersionControlInformation(org.apache.nifi.registry.flow.VersionControlInformation) Optional(java.util.Optional) LocalPort(org.apache.nifi.connectable.LocalPort) StandardProcessContext(org.apache.nifi.processor.StandardProcessContext) ProcessorNode(org.apache.nifi.controller.ProcessorNode) Revision(org.apache.nifi.web.Revision) Funnel(org.apache.nifi.connectable.Funnel) ControllerServiceNode(org.apache.nifi.controller.service.ControllerServiceNode) ToStringStyle(org.apache.commons.lang3.builder.ToStringStyle) HashMap(java.util.HashMap) CompletableFuture(java.util.concurrent.CompletableFuture) AtomicReference(java.util.concurrent.atomic.AtomicReference) Function(java.util.function.Function) FlowRegistry(org.apache.nifi.registry.flow.FlowRegistry) HashSet(java.util.HashSet) StringEncryptor(org.apache.nifi.encrypt.StringEncryptor) ComparableDataFlow(org.apache.nifi.registry.flow.diff.ComparableDataFlow) Objects.requireNonNull(java.util.Objects.requireNonNull) StandardConfigurationContext(org.apache.nifi.controller.service.StandardConfigurationContext) NarCloseable(org.apache.nifi.nar.NarCloseable) LogLevel(org.apache.nifi.logging.LogLevel) VersionedProcessor(org.apache.nifi.registry.flow.VersionedProcessor) Logger(org.slf4j.Logger) StateManager(org.apache.nifi.components.state.StateManager) RemoteGroupPort(org.apache.nifi.remote.RemoteGroupPort) StandardRemoteProcessGroupPortDescriptor(org.apache.nifi.remote.StandardRemoteProcessGroupPortDescriptor) VersionedFlow(org.apache.nifi.registry.flow.VersionedFlow) VersionedControllerService(org.apache.nifi.registry.flow.VersionedControllerService) TimeUnit(java.util.concurrent.TimeUnit) ComponentVariableRegistry(org.apache.nifi.registry.ComponentVariableRegistry) FlowDifference(org.apache.nifi.registry.flow.diff.FlowDifference) VersionedPropertyDescriptor(org.apache.nifi.registry.flow.VersionedPropertyDescriptor) Collections(java.util.Collections) LogRepositoryFactory(org.apache.nifi.logging.LogRepositoryFactory) VersionedFunnel(org.apache.nifi.registry.flow.VersionedFunnel) Funnel(org.apache.nifi.connectable.Funnel) Set(java.util.Set) LinkedHashSet(java.util.LinkedHashSet) HashSet(java.util.HashSet) HashMap(java.util.HashMap) FlowRegistry(org.apache.nifi.registry.flow.FlowRegistry) HashCodeBuilder(org.apache.commons.lang3.builder.HashCodeBuilder) ToStringBuilder(org.apache.commons.lang3.builder.ToStringBuilder) RootGroupPort(org.apache.nifi.remote.RootGroupPort) VersionedRemoteGroupPort(org.apache.nifi.registry.flow.VersionedRemoteGroupPort) Port(org.apache.nifi.connectable.Port) VersionedPort(org.apache.nifi.registry.flow.VersionedPort) LocalPort(org.apache.nifi.connectable.LocalPort) RemoteGroupPort(org.apache.nifi.remote.RemoteGroupPort) VersionedProcessGroup(org.apache.nifi.registry.flow.VersionedProcessGroup) VersionedLabel(org.apache.nifi.registry.flow.VersionedLabel) Label(org.apache.nifi.controller.label.Label) ProcessorNode(org.apache.nifi.controller.ProcessorNode) VersionedFlowState(org.apache.nifi.registry.flow.VersionedFlowState) LinkedHashSet(java.util.LinkedHashSet) HashSet(java.util.HashSet) VersionedRemoteProcessGroup(org.apache.nifi.registry.flow.VersionedRemoteProcessGroup) Position(org.apache.nifi.connectable.Position) Connection(org.apache.nifi.connectable.Connection) VersionedConnection(org.apache.nifi.registry.flow.VersionedConnection) VersionedPort(org.apache.nifi.registry.flow.VersionedPort) VersionedControllerService(org.apache.nifi.registry.flow.VersionedControllerService) VersionedFunnel(org.apache.nifi.registry.flow.VersionedFunnel) VersionedFlowCoordinates(org.apache.nifi.registry.flow.VersionedFlowCoordinates) VersionedLabel(org.apache.nifi.registry.flow.VersionedLabel) StandardVersionControlInformation(org.apache.nifi.registry.flow.StandardVersionControlInformation) VersionControlInformation(org.apache.nifi.registry.flow.VersionControlInformation) ControllerServiceNode(org.apache.nifi.controller.service.ControllerServiceNode) VersionedRemoteProcessGroup(org.apache.nifi.registry.flow.VersionedRemoteProcessGroup) Relationship(org.apache.nifi.processor.Relationship) VersionedProcessGroup(org.apache.nifi.registry.flow.VersionedProcessGroup) VersionedRemoteProcessGroup(org.apache.nifi.registry.flow.VersionedRemoteProcessGroup) VersionedConnection(org.apache.nifi.registry.flow.VersionedConnection) Map(java.util.Map) HashMap(java.util.HashMap) VersionedProcessor(org.apache.nifi.registry.flow.VersionedProcessor)

Example 3 with VersionedRemoteProcessGroup

use of org.apache.nifi.registry.flow.VersionedRemoteProcessGroup in project nifi-minifi by apache.

the class VersionedProcessGroupEnricher method enrich.

public void enrich(final VersionedProcessGroup versionedProcessGroup) {
    List<VersionedProcessGroup> allVersionedProcessGroups = getAllVersionedProcessGroups(versionedProcessGroup);
    Set<VersionedRemoteProcessGroup> remoteProcessGroups = getAll(allVersionedProcessGroups, VersionedProcessGroup::getRemoteProcessGroups).collect(Collectors.toSet());
    Map<String, String> connectableNameMap = getAll(allVersionedProcessGroups, VersionedProcessGroup::getProcessors).collect(Collectors.toMap(VersionedComponent::getIdentifier, VersionedComponent::getName));
    Map<String, String> rpgIdToTargetIdMap = new HashMap<>();
    for (VersionedRemoteProcessGroup remoteProcessGroup : remoteProcessGroups) {
        final Set<VersionedRemoteGroupPort> rpgInputPorts = nullToEmpty(remoteProcessGroup.getInputPorts());
        final Set<VersionedRemoteGroupPort> rpgOutputPorts = nullToEmpty(remoteProcessGroup.getOutputPorts());
        // Map all port DTOs to their respective targetIds
        rpgIdToTargetIdMap.putAll(Stream.concat(rpgInputPorts.stream(), rpgOutputPorts.stream()).collect(Collectors.toMap(VersionedRemoteGroupPort::getIdentifier, VersionedRemoteGroupPort::getTargetId)));
        addConnectables(connectableNameMap, rpgInputPorts, VersionedRemoteGroupPort::getIdentifier, VersionedRemoteGroupPort::getIdentifier);
        addConnectables(connectableNameMap, rpgOutputPorts, VersionedRemoteGroupPort::getIdentifier, VersionedRemoteGroupPort::getIdentifier);
    }
    addConnectables(connectableNameMap, getAll(allVersionedProcessGroups, VersionedProcessGroup::getInputPorts).collect(Collectors.toList()), VersionedPort::getIdentifier, VersionedPort::getName);
    addConnectables(connectableNameMap, getAll(allVersionedProcessGroups, VersionedProcessGroup::getOutputPorts).collect(Collectors.toList()), VersionedPort::getIdentifier, VersionedPort::getName);
    final Set<VersionedConnection> connections = getAll(allVersionedProcessGroups, VersionedProcessGroup::getConnections).collect(Collectors.toSet());
    // Enrich connection endpoints using known names and overriding with targetIds for remote ports
    for (VersionedConnection connection : connections) {
        setName(connectableNameMap, connection.getSource(), rpgIdToTargetIdMap);
        setName(connectableNameMap, connection.getDestination(), rpgIdToTargetIdMap);
    }
    // Override any ids that are for Remote Ports to use their target Ids where available
    connections.stream().flatMap(connectionDTO -> Stream.of(connectionDTO.getSource(), connectionDTO.getDestination())).filter(connectable -> (connectable.getType() == ConnectableComponentType.REMOTE_OUTPUT_PORT || connectable.getType() == ConnectableComponentType.REMOTE_INPUT_PORT)).forEach(connectable -> connectable.setId(Optional.ofNullable(rpgIdToTargetIdMap.get(connectable.getId())).orElse(connectable.getId())));
    // Establish unique names for connections
    for (VersionedConnection connection : connections) {
        if (StringUtil.isNullOrEmpty(connection.getName())) {
            StringBuilder name = new StringBuilder();
            ConnectableComponent connectionSource = connection.getSource();
            name.append(determineValueForConnectable(connectionSource, rpgIdToTargetIdMap));
            name.append("/");
            if (connection.getSelectedRelationships() != null && connection.getSelectedRelationships().size() > 0) {
                name.append(connection.getSelectedRelationships().iterator().next());
            }
            name.append("/");
            ConnectableComponent connectionDestination = connection.getDestination();
            name.append(determineValueForConnectable(connectionDestination, rpgIdToTargetIdMap));
            connection.setName(name.toString());
        }
    }
    nullToEmpty(versionedProcessGroup.getProcessGroups()).stream().forEach(pg -> enrich(pg));
}
Also used : ConnectableComponent(org.apache.nifi.registry.flow.ConnectableComponent) Collection(java.util.Collection) Set(java.util.Set) HashMap(java.util.HashMap) StringUtils(org.apache.commons.lang3.StringUtils) VersionedRemoteGroupPort(org.apache.nifi.registry.flow.VersionedRemoteGroupPort) Function(java.util.function.Function) Collectors(java.util.stream.Collectors) ArrayList(java.util.ArrayList) Objects(java.util.Objects) List(java.util.List) Stream(java.util.stream.Stream) StringUtil(org.apache.nifi.minifi.commons.schema.common.StringUtil) VersionedProcessGroup(org.apache.nifi.registry.flow.VersionedProcessGroup) Map(java.util.Map) VersionedPort(org.apache.nifi.registry.flow.VersionedPort) VersionedRemoteProcessGroup(org.apache.nifi.registry.flow.VersionedRemoteProcessGroup) Optional(java.util.Optional) CollectionUtil.nullToEmpty(org.apache.nifi.minifi.commons.schema.common.CollectionUtil.nullToEmpty) VersionedComponent(org.apache.nifi.registry.flow.VersionedComponent) VersionedConnection(org.apache.nifi.registry.flow.VersionedConnection) ConnectableComponentType(org.apache.nifi.registry.flow.ConnectableComponentType) HashMap(java.util.HashMap) VersionedProcessGroup(org.apache.nifi.registry.flow.VersionedProcessGroup) VersionedRemoteGroupPort(org.apache.nifi.registry.flow.VersionedRemoteGroupPort) VersionedPort(org.apache.nifi.registry.flow.VersionedPort) ConnectableComponent(org.apache.nifi.registry.flow.ConnectableComponent) VersionedRemoteProcessGroup(org.apache.nifi.registry.flow.VersionedRemoteProcessGroup) VersionedConnection(org.apache.nifi.registry.flow.VersionedConnection)

Example 4 with VersionedRemoteProcessGroup

use of org.apache.nifi.registry.flow.VersionedRemoteProcessGroup in project nifi by apache.

the class StandardProcessGroup method addRemoteProcessGroup.

private RemoteProcessGroup addRemoteProcessGroup(final ProcessGroup destination, final VersionedRemoteProcessGroup proposed, final String componentIdSeed) {
    final RemoteProcessGroup rpg = flowController.createRemoteProcessGroup(generateUuid(proposed.getIdentifier(), destination.getIdentifier(), componentIdSeed), proposed.getTargetUris());
    rpg.setVersionedComponentId(proposed.getIdentifier());
    destination.addRemoteProcessGroup(rpg);
    updateRemoteProcessGroup(rpg, proposed, componentIdSeed);
    return rpg;
}
Also used : VersionedRemoteProcessGroup(org.apache.nifi.registry.flow.VersionedRemoteProcessGroup)

Example 5 with VersionedRemoteProcessGroup

use of org.apache.nifi.registry.flow.VersionedRemoteProcessGroup in project nifi by apache.

the class NiFiRegistryFlowMapper method mapRemoteProcessGroup.

public VersionedRemoteProcessGroup mapRemoteProcessGroup(final RemoteProcessGroup remoteGroup) {
    final VersionedRemoteProcessGroup rpg = new InstantiatedVersionedRemoteProcessGroup(remoteGroup.getIdentifier(), remoteGroup.getProcessGroupIdentifier());
    rpg.setIdentifier(getId(remoteGroup.getVersionedComponentId(), remoteGroup.getIdentifier()));
    rpg.setGroupIdentifier(getGroupId(remoteGroup.getProcessGroupIdentifier()));
    rpg.setComments(remoteGroup.getComments());
    rpg.setCommunicationsTimeout(remoteGroup.getCommunicationsTimeout());
    rpg.setLocalNetworkInterface(remoteGroup.getNetworkInterface());
    rpg.setName(remoteGroup.getName());
    rpg.setInputPorts(remoteGroup.getInputPorts().stream().map(port -> mapRemotePort(port, ComponentType.REMOTE_INPUT_PORT)).collect(Collectors.toSet()));
    rpg.setOutputPorts(remoteGroup.getOutputPorts().stream().map(port -> mapRemotePort(port, ComponentType.REMOTE_OUTPUT_PORT)).collect(Collectors.toSet()));
    rpg.setPosition(mapPosition(remoteGroup.getPosition()));
    rpg.setProxyHost(remoteGroup.getProxyHost());
    rpg.setProxyPort(remoteGroup.getProxyPort());
    rpg.setProxyUser(remoteGroup.getProxyUser());
    rpg.setTargetUri(remoteGroup.getTargetUri());
    rpg.setTargetUris(remoteGroup.getTargetUris());
    rpg.setTransportProtocol(remoteGroup.getTransportProtocol().name());
    rpg.setYieldDuration(remoteGroup.getYieldDuration());
    return rpg;
}
Also used : VersionedRemoteProcessGroup(org.apache.nifi.registry.flow.VersionedRemoteProcessGroup)

Aggregations

VersionedRemoteProcessGroup (org.apache.nifi.registry.flow.VersionedRemoteProcessGroup)5 ArrayList (java.util.ArrayList)3 HashMap (java.util.HashMap)3 List (java.util.List)3 Map (java.util.Map)3 Objects (java.util.Objects)3 Optional (java.util.Optional)3 Set (java.util.Set)3 Function (java.util.function.Function)3 Collectors (java.util.stream.Collectors)3 StringUtils (org.apache.commons.lang3.StringUtils)3 ConnectableComponent (org.apache.nifi.registry.flow.ConnectableComponent)3 VersionedComponent (org.apache.nifi.registry.flow.VersionedComponent)3 VersionedConnection (org.apache.nifi.registry.flow.VersionedConnection)3 VersionedPort (org.apache.nifi.registry.flow.VersionedPort)3 VersionedProcessGroup (org.apache.nifi.registry.flow.VersionedProcessGroup)3 VersionedRemoteGroupPort (org.apache.nifi.registry.flow.VersionedRemoteGroupPort)3 IOException (java.io.IOException)2 URL (java.net.URL)2 StandardCharsets (java.nio.charset.StandardCharsets)2