Examples with Port org.apache.nifi.connectable.Port used on opensource projects

Example 36 with Port

use of org.apache.nifi.connectable.Port in project nifi by apache.

the class StandardProcessGroup method removeOutputPort.

@Override
public void removeOutputPort(final Port port) {
    writeLock.lock();
    try {
        final Port toRemove = outputPorts.get(requireNonNull(port).getIdentifier());
        toRemove.verifyCanDelete();
        if (port.isRunning()) {
            stopOutputPort(port);
        }
        if (!toRemove.getConnections().isEmpty()) {
            throw new IllegalStateException(port.getIdentifier() + " cannot be removed until its connections are removed");
        }
        final Port removed = outputPorts.remove(port.getIdentifier());
        if (removed == null) {
            throw new IllegalStateException(port.getIdentifier() + " is not an Output Port of this Process Group");
        }
        onComponentModified();
        flowController.onOutputPortRemoved(port);
        LOG.info("Output Port {} removed from flow", port);
    } finally {
        writeLock.unlock();
    }
}

Example 37 with Port

use of org.apache.nifi.connectable.Port in project nifi by apache.

the class StandardProcessGroup method getConnectable.

@Override
public Connectable getConnectable(final String id) {
    readLock.lock();
    try {
        final ProcessorNode node = processors.get(id);
        if (node != null) {
            return node;
        }
        final Port inputPort = inputPorts.get(id);
        if (inputPort != null) {
            return inputPort;
        }
        final Port outputPort = outputPorts.get(id);
        if (outputPort != null) {
            return outputPort;
        }
        final Funnel funnel = funnels.get(id);
        if (funnel != null) {
            return funnel;
        }
        return null;
    } finally {
        readLock.unlock();
    }
}

Example 38 with Port

use of org.apache.nifi.connectable.Port in project nifi by apache.

the class StandardProcessGroup method updateProcessGroup.

private void updateProcessGroup(final ProcessGroup group, final VersionedProcessGroup proposed, final String componentIdSeed, final Set<String> updatedVersionedComponentIds, final boolean updatePosition, final boolean updateName, final boolean updateDescendantVersionedGroups, final Set<String> variablesToSkip) throws ProcessorInstantiationException {
    group.setComments(proposed.getComments());
    if (updateName) {
        group.setName(proposed.getName());
    }
    if (updatePosition && proposed.getPosition() != null) {
        group.setPosition(new Position(proposed.getPosition().getX(), proposed.getPosition().getY()));
    }
    // Determine which variables have been added/removed and add/remove them from this group's variable registry.
    // We don't worry about if a variable value has changed, because variables are designed to be 'environment specific.'
    // As a result, once imported, we won't update variables to match the remote flow, but we will add any missing variables
    // and remove any variables that are no longer part of the remote flow.
    final Set<String> existingVariableNames = group.getVariableRegistry().getVariableMap().keySet().stream().map(VariableDescriptor::getName).collect(Collectors.toSet());
    final Map<String, String> updatedVariableMap = new HashMap<>();
    // If any new variables exist in the proposed flow, add those to the variable registry.
    for (final Map.Entry<String, String> entry : proposed.getVariables().entrySet()) {
        if (!existingVariableNames.contains(entry.getKey()) && !variablesToSkip.contains(entry.getKey())) {
            updatedVariableMap.put(entry.getKey(), entry.getValue());
        }
    }
    group.setVariables(updatedVariableMap);
    final VersionedFlowCoordinates remoteCoordinates = proposed.getVersionedFlowCoordinates();
    if (remoteCoordinates == null) {
        group.disconnectVersionControl(false);
    } else {
        final String registryId = flowController.getFlowRegistryClient().getFlowRegistryId(remoteCoordinates.getRegistryUrl());
        final String bucketId = remoteCoordinates.getBucketId();
        final String flowId = remoteCoordinates.getFlowId();
        final int version = remoteCoordinates.getVersion();
        final FlowRegistry flowRegistry = flowController.getFlowRegistryClient().getFlowRegistry(registryId);
        final String registryName = flowRegistry == null ? registryId : flowRegistry.getName();
        final VersionedFlowState flowState = remoteCoordinates.getLatest() ? VersionedFlowState.UP_TO_DATE : VersionedFlowState.STALE;
        final VersionControlInformation vci = new StandardVersionControlInformation.Builder().registryId(registryId).registryName(registryName).bucketId(bucketId).bucketName(bucketId).flowId(flowId).flowName(flowId).version(version).flowSnapshot(proposed).status(new StandardVersionedFlowStatus(flowState, flowState.getDescription())).build();
        group.setVersionControlInformation(vci, Collections.emptyMap());
    }
    // Controller Services
    // Controller Services have to be handled a bit differently than other components. This is because Processors and Controller
    // Services may reference other Controller Services. Since we may be adding Service A, which depends on Service B, before adding
    // Service B, we need to ensure that we create all Controller Services first and then call updateControllerService for each
    // Controller Service. This way, we ensure that all services have been created before setting the properties. This allows us to
    // properly obtain the correct mapping of Controller Service VersionedComponentID to Controller Service instance id.
    final Map<String, ControllerServiceNode> servicesByVersionedId = group.getControllerServices(false).stream().collect(Collectors.toMap(component -> component.getVersionedComponentId().orElse(component.getIdentifier()), Function.identity()));
    final Set<String> controllerServicesRemoved = new HashSet<>(servicesByVersionedId.keySet());
    final Map<ControllerServiceNode, VersionedControllerService> services = new HashMap<>();
    // Add any Controller Service that does not yet exist.
    for (final VersionedControllerService proposedService : proposed.getControllerServices()) {
        ControllerServiceNode service = servicesByVersionedId.get(proposedService.getIdentifier());
        if (service == null) {
            service = addControllerService(group, proposedService, componentIdSeed);
            LOG.info("Added {} to {}", service, this);
        }
        services.put(service, proposedService);
    }
    // Update all of the Controller Services to match the VersionedControllerService
    for (final Map.Entry<ControllerServiceNode, VersionedControllerService> entry : services.entrySet()) {
        final ControllerServiceNode service = entry.getKey();
        final VersionedControllerService proposedService = entry.getValue();
        if (updatedVersionedComponentIds.contains(proposedService.getIdentifier())) {
            updateControllerService(service, proposedService);
            LOG.info("Updated {}", service);
        }
        controllerServicesRemoved.remove(proposedService.getIdentifier());
    }
    // Child groups
    final Map<String, ProcessGroup> childGroupsByVersionedId = group.getProcessGroups().stream().collect(Collectors.toMap(component -> component.getVersionedComponentId().orElse(component.getIdentifier()), Function.identity()));
    final Set<String> childGroupsRemoved = new HashSet<>(childGroupsByVersionedId.keySet());
    for (final VersionedProcessGroup proposedChildGroup : proposed.getProcessGroups()) {
        final ProcessGroup childGroup = childGroupsByVersionedId.get(proposedChildGroup.getIdentifier());
        final VersionedFlowCoordinates childCoordinates = proposedChildGroup.getVersionedFlowCoordinates();
        if (childGroup == null) {
            final ProcessGroup added = addProcessGroup(group, proposedChildGroup, componentIdSeed, variablesToSkip);
            flowController.onProcessGroupAdded(added);
            added.findAllRemoteProcessGroups().stream().forEach(RemoteProcessGroup::initialize);
            LOG.info("Added {} to {}", added, this);
        } else if (childCoordinates == null || updateDescendantVersionedGroups) {
            updateProcessGroup(childGroup, proposedChildGroup, componentIdSeed, updatedVersionedComponentIds, true, true, updateDescendantVersionedGroups, variablesToSkip);
            LOG.info("Updated {}", childGroup);
        }
        childGroupsRemoved.remove(proposedChildGroup.getIdentifier());
    }
    // Funnels
    final Map<String, Funnel> funnelsByVersionedId = group.getFunnels().stream().collect(Collectors.toMap(component -> component.getVersionedComponentId().orElse(component.getIdentifier()), Function.identity()));
    final Set<String> funnelsRemoved = new HashSet<>(funnelsByVersionedId.keySet());
    for (final VersionedFunnel proposedFunnel : proposed.getFunnels()) {
        final Funnel funnel = funnelsByVersionedId.get(proposedFunnel.getIdentifier());
        if (funnel == null) {
            final Funnel added = addFunnel(group, proposedFunnel, componentIdSeed);
            flowController.onFunnelAdded(added);
            LOG.info("Added {} to {}", added, this);
        } else if (updatedVersionedComponentIds.contains(proposedFunnel.getIdentifier())) {
            updateFunnel(funnel, proposedFunnel);
            LOG.info("Updated {}", funnel);
        } else {
            funnel.setPosition(new Position(proposedFunnel.getPosition().getX(), proposedFunnel.getPosition().getY()));
        }
        funnelsRemoved.remove(proposedFunnel.getIdentifier());
    }
    // Input Ports
    final Map<String, Port> inputPortsByVersionedId = group.getInputPorts().stream().collect(Collectors.toMap(component -> component.getVersionedComponentId().orElse(component.getIdentifier()), Function.identity()));
    final Set<String> inputPortsRemoved = new HashSet<>(inputPortsByVersionedId.keySet());
    for (final VersionedPort proposedPort : proposed.getInputPorts()) {
        final Port port = inputPortsByVersionedId.get(proposedPort.getIdentifier());
        if (port == null) {
            final Port added = addInputPort(group, proposedPort, componentIdSeed);
            flowController.onInputPortAdded(added);
            LOG.info("Added {} to {}", added, this);
        } else if (updatedVersionedComponentIds.contains(proposedPort.getIdentifier())) {
            updatePort(port, proposedPort);
            LOG.info("Updated {}", port);
        } else {
            port.setPosition(new Position(proposedPort.getPosition().getX(), proposedPort.getPosition().getY()));
        }
        inputPortsRemoved.remove(proposedPort.getIdentifier());
    }
    // Output Ports
    final Map<String, Port> outputPortsByVersionedId = group.getOutputPorts().stream().collect(Collectors.toMap(component -> component.getVersionedComponentId().orElse(component.getIdentifier()), Function.identity()));
    final Set<String> outputPortsRemoved = new HashSet<>(outputPortsByVersionedId.keySet());
    for (final VersionedPort proposedPort : proposed.getOutputPorts()) {
        final Port port = outputPortsByVersionedId.get(proposedPort.getIdentifier());
        if (port == null) {
            final Port added = addOutputPort(group, proposedPort, componentIdSeed);
            flowController.onOutputPortAdded(added);
            LOG.info("Added {} to {}", added, this);
        } else if (updatedVersionedComponentIds.contains(proposedPort.getIdentifier())) {
            updatePort(port, proposedPort);
            LOG.info("Updated {}", port);
        } else {
            port.setPosition(new Position(proposedPort.getPosition().getX(), proposedPort.getPosition().getY()));
        }
        outputPortsRemoved.remove(proposedPort.getIdentifier());
    }
    // Labels
    final Map<String, Label> labelsByVersionedId = group.getLabels().stream().collect(Collectors.toMap(component -> component.getVersionedComponentId().orElse(component.getIdentifier()), Function.identity()));
    final Set<String> labelsRemoved = new HashSet<>(labelsByVersionedId.keySet());
    for (final VersionedLabel proposedLabel : proposed.getLabels()) {
        final Label label = labelsByVersionedId.get(proposedLabel.getIdentifier());
        if (label == null) {
            final Label added = addLabel(group, proposedLabel, componentIdSeed);
            LOG.info("Added {} to {}", added, this);
        } else if (updatedVersionedComponentIds.contains(proposedLabel.getIdentifier())) {
            updateLabel(label, proposedLabel);
            LOG.info("Updated {}", label);
        } else {
            label.setPosition(new Position(proposedLabel.getPosition().getX(), proposedLabel.getPosition().getY()));
        }
        labelsRemoved.remove(proposedLabel.getIdentifier());
    }
    // Processors
    final Map<String, ProcessorNode> processorsByVersionedId = group.getProcessors().stream().collect(Collectors.toMap(component -> component.getVersionedComponentId().orElse(component.getIdentifier()), Function.identity()));
    final Set<String> processorsRemoved = new HashSet<>(processorsByVersionedId.keySet());
    final Map<ProcessorNode, Set<Relationship>> autoTerminatedRelationships = new HashMap<>();
    for (final VersionedProcessor proposedProcessor : proposed.getProcessors()) {
        final ProcessorNode processor = processorsByVersionedId.get(proposedProcessor.getIdentifier());
        if (processor == null) {
            final ProcessorNode added = addProcessor(group, proposedProcessor, componentIdSeed);
            flowController.onProcessorAdded(added);
            final Set<Relationship> proposedAutoTerminated = proposedProcessor.getAutoTerminatedRelationships() == null ? Collections.emptySet() : proposedProcessor.getAutoTerminatedRelationships().stream().map(relName -> added.getRelationship(relName)).collect(Collectors.toSet());
            autoTerminatedRelationships.put(added, proposedAutoTerminated);
            LOG.info("Added {} to {}", added, this);
        } else if (updatedVersionedComponentIds.contains(proposedProcessor.getIdentifier())) {
            updateProcessor(processor, proposedProcessor);
            final Set<Relationship> proposedAutoTerminated = proposedProcessor.getAutoTerminatedRelationships() == null ? Collections.emptySet() : proposedProcessor.getAutoTerminatedRelationships().stream().map(relName -> processor.getRelationship(relName)).collect(Collectors.toSet());
            if (!processor.getAutoTerminatedRelationships().equals(proposedAutoTerminated)) {
                autoTerminatedRelationships.put(processor, proposedAutoTerminated);
            }
            LOG.info("Updated {}", processor);
        } else {
            processor.setPosition(new Position(proposedProcessor.getPosition().getX(), proposedProcessor.getPosition().getY()));
        }
        processorsRemoved.remove(proposedProcessor.getIdentifier());
    }
    // Remote Groups
    final Map<String, RemoteProcessGroup> rpgsByVersionedId = group.getRemoteProcessGroups().stream().collect(Collectors.toMap(component -> component.getVersionedComponentId().orElse(component.getIdentifier()), Function.identity()));
    final Set<String> rpgsRemoved = new HashSet<>(rpgsByVersionedId.keySet());
    for (final VersionedRemoteProcessGroup proposedRpg : proposed.getRemoteProcessGroups()) {
        final RemoteProcessGroup rpg = rpgsByVersionedId.get(proposedRpg.getIdentifier());
        if (rpg == null) {
            final RemoteProcessGroup added = addRemoteProcessGroup(group, proposedRpg, componentIdSeed);
            LOG.info("Added {} to {}", added, this);
        } else if (updatedVersionedComponentIds.contains(proposedRpg.getIdentifier())) {
            updateRemoteProcessGroup(rpg, proposedRpg, componentIdSeed);
            LOG.info("Updated {}", rpg);
        } else {
            rpg.setPosition(new Position(proposedRpg.getPosition().getX(), proposedRpg.getPosition().getY()));
        }
        rpgsRemoved.remove(proposedRpg.getIdentifier());
    }
    // Connections
    final Map<String, Connection> connectionsByVersionedId = group.getConnections().stream().collect(Collectors.toMap(component -> component.getVersionedComponentId().orElse(component.getIdentifier()), Function.identity()));
    final Set<String> connectionsRemoved = new HashSet<>(connectionsByVersionedId.keySet());
    for (final VersionedConnection proposedConnection : proposed.getConnections()) {
        final Connection connection = connectionsByVersionedId.get(proposedConnection.getIdentifier());
        if (connection == null) {
            final Connection added = addConnection(group, proposedConnection, componentIdSeed);
            flowController.onConnectionAdded(added);
            LOG.info("Added {} to {}", added, this);
        } else if (isUpdateable(connection)) {
            // If the connection needs to be updated, then the source and destination will already have
            // been stopped (else, the validation above would fail). So if the source or the destination is running,
            // then we know that we don't need to update the connection.
            updateConnection(connection, proposedConnection);
            LOG.info("Updated {}", connection);
        }
        connectionsRemoved.remove(proposedConnection.getIdentifier());
    }
    // to remove a component if it has a connection going to it!
    for (final String removedVersionedId : connectionsRemoved) {
        final Connection connection = connectionsByVersionedId.get(removedVersionedId);
        LOG.info("Removing {} from {}", connection, group);
        group.removeConnection(connection);
        flowController.onConnectionRemoved(connection);
    }
    // Once the appropriate connections have been removed, we may now update Processors' auto-terminated relationships.
    // We cannot do this above, in the 'updateProcessor' call because if a connection is removed and changed to auto-terminated,
    // then updating this in the updateProcessor call above would attempt to set the Relationship to being auto-terminated while a
    // Connection for that relationship exists. This will throw an Exception.
    autoTerminatedRelationships.forEach((proc, rels) -> proc.setAutoTerminatedRelationships(rels));
    // Remove all controller services no longer in use
    for (final String removedVersionedId : controllerServicesRemoved) {
        final ControllerServiceNode service = servicesByVersionedId.get(removedVersionedId);
        LOG.info("Removing {} from {}", service, group);
        // Must remove Controller Service through Flow Controller in order to remove from cache
        flowController.removeControllerService(service);
    }
    for (final String removedVersionedId : funnelsRemoved) {
        final Funnel funnel = funnelsByVersionedId.get(removedVersionedId);
        LOG.info("Removing {} from {}", funnel, group);
        group.removeFunnel(funnel);
    }
    for (final String removedVersionedId : inputPortsRemoved) {
        final Port port = inputPortsByVersionedId.get(removedVersionedId);
        LOG.info("Removing {} from {}", port, group);
        group.removeInputPort(port);
    }
    for (final String removedVersionedId : outputPortsRemoved) {
        final Port port = outputPortsByVersionedId.get(removedVersionedId);
        LOG.info("Removing {} from {}", port, group);
        group.removeOutputPort(port);
    }
    for (final String removedVersionedId : labelsRemoved) {
        final Label label = labelsByVersionedId.get(removedVersionedId);
        LOG.info("Removing {} from {}", label, group);
        group.removeLabel(label);
    }
    for (final String removedVersionedId : processorsRemoved) {
        final ProcessorNode processor = processorsByVersionedId.get(removedVersionedId);
        LOG.info("Removing {} from {}", processor, group);
        group.removeProcessor(processor);
    }
    for (final String removedVersionedId : rpgsRemoved) {
        final RemoteProcessGroup rpg = rpgsByVersionedId.get(removedVersionedId);
        LOG.info("Removing {} from {}", rpg, group);
        group.removeRemoteProcessGroup(rpg);
    }
    for (final String removedVersionedId : childGroupsRemoved) {
        final ProcessGroup childGroup = childGroupsByVersionedId.get(removedVersionedId);
        LOG.info("Removing {} from {}", childGroup, group);
        group.removeProcessGroup(childGroup);
    }
}

Example 39 with Port

use of org.apache.nifi.connectable.Port in project nifi by apache.

the class NiFiRegistryFlowMapper method mapGroup.

private InstantiatedVersionedProcessGroup mapGroup(final ProcessGroup group, final ControllerServiceProvider serviceLookup, final FlowRegistryClient registryClient, final boolean topLevel, final boolean mapDescendantVersionedFlows) {
    final InstantiatedVersionedProcessGroup versionedGroup = new InstantiatedVersionedProcessGroup(group.getIdentifier(), group.getProcessGroupIdentifier());
    versionedGroup.setIdentifier(getId(group.getVersionedComponentId(), group.getIdentifier()));
    versionedGroup.setGroupIdentifier(getGroupId(group.getProcessGroupIdentifier()));
    versionedGroup.setName(group.getName());
    versionedGroup.setComments(group.getComments());
    versionedGroup.setPosition(mapPosition(group.getPosition()));
    // only for a child group that is itself version controlled.
    if (!topLevel) {
        final VersionControlInformation versionControlInfo = group.getVersionControlInformation();
        if (versionControlInfo != null) {
            final VersionedFlowCoordinates coordinates = new VersionedFlowCoordinates();
            final String registryId = versionControlInfo.getRegistryIdentifier();
            final FlowRegistry registry = registryClient.getFlowRegistry(registryId);
            if (registry == null) {
                throw new IllegalStateException("Process Group refers to a Flow Registry with ID " + registryId + " but no Flow Registry exists with that ID. Cannot resolve to a URL.");
            }
            coordinates.setRegistryUrl(registry.getURL());
            coordinates.setBucketId(versionControlInfo.getBucketIdentifier());
            coordinates.setFlowId(versionControlInfo.getFlowIdentifier());
            coordinates.setVersion(versionControlInfo.getVersion());
            versionedGroup.setVersionedFlowCoordinates(coordinates);
            // Otherwise, we will not be able to lookup the port when connecting to it.
            for (final Port port : group.getInputPorts()) {
                getId(port.getVersionedComponentId(), port.getIdentifier());
            }
            for (final Port port : group.getOutputPorts()) {
                getId(port.getVersionedComponentId(), port.getIdentifier());
            }
            // because the contents are remotely managed and not part of the versioning of this Process Group
            if (!mapDescendantVersionedFlows) {
                return versionedGroup;
            }
        }
    }
    versionedGroup.setControllerServices(group.getControllerServices(false).stream().map(service -> mapControllerService(service, serviceLookup)).collect(Collectors.toCollection(LinkedHashSet::new)));
    versionedGroup.setFunnels(group.getFunnels().stream().map(this::mapFunnel).collect(Collectors.toCollection(LinkedHashSet::new)));
    versionedGroup.setInputPorts(group.getInputPorts().stream().map(this::mapPort).collect(Collectors.toCollection(LinkedHashSet::new)));
    versionedGroup.setOutputPorts(group.getOutputPorts().stream().map(this::mapPort).collect(Collectors.toCollection(LinkedHashSet::new)));
    versionedGroup.setLabels(group.getLabels().stream().map(this::mapLabel).collect(Collectors.toCollection(LinkedHashSet::new)));
    versionedGroup.setProcessors(group.getProcessors().stream().map(processor -> mapProcessor(processor, serviceLookup)).collect(Collectors.toCollection(LinkedHashSet::new)));
    versionedGroup.setRemoteProcessGroups(group.getRemoteProcessGroups().stream().map(this::mapRemoteProcessGroup).collect(Collectors.toCollection(LinkedHashSet::new)));
    versionedGroup.setProcessGroups(group.getProcessGroups().stream().map(grp -> mapGroup(grp, serviceLookup, registryClient, false, mapDescendantVersionedFlows)).collect(Collectors.toCollection(LinkedHashSet::new)));
    versionedGroup.setConnections(group.getConnections().stream().map(this::mapConnection).collect(Collectors.toCollection(LinkedHashSet::new)));
    versionedGroup.setVariables(group.getVariableRegistry().getVariableMap().entrySet().stream().collect(Collectors.toMap(entry -> entry.getKey().getName(), Map.Entry::getValue)));
    return versionedGroup;
}

Example 40 with Port

use of org.apache.nifi.connectable.Port in project nifi by apache.

the class StandardRemoteProcessGroup method startTransmitting.

@Override
public void startTransmitting() {
    writeLock.lock();
    try {
        verifyCanStartTransmitting();
        for (final Port port : getInputPorts()) {
            // Validation is based on connections and whether or not the remote target exists.
            if (port.isValid() && port.hasIncomingConnection()) {
                scheduler.startPort(port);
            }
        }
        for (final Port port : getOutputPorts()) {
            if (port.isValid() && !port.getConnections().isEmpty()) {
                scheduler.startPort(port);
            }
        }
        transmitting.set(true);
    } finally {
        writeLock.unlock();
    }
}