Examples with Connectable org.apache.nifi.connectable.Connectable used on opensource projects

Example 16 with Connectable

use of org.apache.nifi.connectable.Connectable in project nifi by apache.

the class StandardNiFiServiceFacade method getComponentsAffectedByVersionChange.

@Override
public Set<AffectedComponentEntity> getComponentsAffectedByVersionChange(final String processGroupId, final VersionedFlowSnapshot updatedSnapshot, final NiFiUser user) {
    final ProcessGroup group = processGroupDAO.getProcessGroup(processGroupId);
    final NiFiRegistryFlowMapper mapper = new NiFiRegistryFlowMapper();
    final VersionedProcessGroup localContents = mapper.mapProcessGroup(group, controllerFacade.getControllerServiceProvider(), flowRegistryClient, true);
    final ComparableDataFlow localFlow = new StandardComparableDataFlow("Local Flow", localContents);
    final ComparableDataFlow proposedFlow = new StandardComparableDataFlow("Versioned Flow", updatedSnapshot.getFlowContents());
    final Set<String> ancestorGroupServiceIds = getAncestorGroupServiceIds(group);
    final FlowComparator flowComparator = new StandardFlowComparator(localFlow, proposedFlow, ancestorGroupServiceIds, new StaticDifferenceDescriptor());
    final FlowComparison comparison = flowComparator.compare();
    final Set<AffectedComponentEntity> affectedComponents = comparison.getDifferences().stream().filter(// components that are added are not components that will be affected in the local flow.
    difference -> difference.getDifferenceType() != DifferenceType.COMPONENT_ADDED).filter(difference -> difference.getDifferenceType() != DifferenceType.BUNDLE_CHANGED).filter(FlowDifferenceFilters.FILTER_ADDED_REMOVED_REMOTE_PORTS).map(difference -> {
        final VersionedComponent localComponent = difference.getComponentA();
        final String state;
        switch(localComponent.getComponentType()) {
            case CONTROLLER_SERVICE:
                final String serviceId = ((InstantiatedVersionedControllerService) localComponent).getInstanceId();
                state = controllerServiceDAO.getControllerService(serviceId).getState().name();
                break;
            case PROCESSOR:
                final String processorId = ((InstantiatedVersionedProcessor) localComponent).getInstanceId();
                state = processorDAO.getProcessor(processorId).getPhysicalScheduledState().name();
                break;
            case REMOTE_INPUT_PORT:
                final InstantiatedVersionedRemoteGroupPort inputPort = (InstantiatedVersionedRemoteGroupPort) localComponent;
                state = remoteProcessGroupDAO.getRemoteProcessGroup(inputPort.getInstanceGroupId()).getInputPort(inputPort.getInstanceId()).getScheduledState().name();
                break;
            case REMOTE_OUTPUT_PORT:
                final InstantiatedVersionedRemoteGroupPort outputPort = (InstantiatedVersionedRemoteGroupPort) localComponent;
                state = remoteProcessGroupDAO.getRemoteProcessGroup(outputPort.getInstanceGroupId()).getOutputPort(outputPort.getInstanceId()).getScheduledState().name();
                break;
            default:
                state = null;
                break;
        }
        return createAffectedComponentEntity((InstantiatedVersionedComponent) localComponent, localComponent.getComponentType().name(), state, user);
    }).collect(Collectors.toCollection(HashSet::new));
    for (final FlowDifference difference : comparison.getDifferences()) {
        // Ignore these as local differences for now because we can't do anything with it
        if (difference.getDifferenceType() == DifferenceType.BUNDLE_CHANGED) {
            continue;
        }
        // Ignore differences for adding remote ports
        if (FlowDifferenceFilters.isAddedOrRemovedRemotePort(difference)) {
            continue;
        }
        final VersionedComponent localComponent = difference.getComponentA();
        if (localComponent == null) {
            continue;
        }
        // If any Process Group is removed, consider all components below that Process Group as an affected component
        if (difference.getDifferenceType() == DifferenceType.COMPONENT_REMOVED && localComponent.getComponentType() == org.apache.nifi.registry.flow.ComponentType.PROCESS_GROUP) {
            final String localGroupId = ((InstantiatedVersionedProcessGroup) localComponent).getInstanceId();
            final ProcessGroup localGroup = processGroupDAO.getProcessGroup(localGroupId);
            localGroup.findAllProcessors().stream().map(comp -> createAffectedComponentEntity(comp, user)).forEach(affectedComponents::add);
            localGroup.findAllFunnels().stream().map(comp -> createAffectedComponentEntity(comp, user)).forEach(affectedComponents::add);
            localGroup.findAllInputPorts().stream().map(comp -> createAffectedComponentEntity(comp, user)).forEach(affectedComponents::add);
            localGroup.findAllOutputPorts().stream().map(comp -> createAffectedComponentEntity(comp, user)).forEach(affectedComponents::add);
            localGroup.findAllRemoteProcessGroups().stream().flatMap(rpg -> Stream.concat(rpg.getInputPorts().stream(), rpg.getOutputPorts().stream())).map(comp -> createAffectedComponentEntity(comp, user)).forEach(affectedComponents::add);
            localGroup.findAllControllerServices().stream().map(comp -> createAffectedComponentEntity(comp, user)).forEach(affectedComponents::add);
        }
        if (localComponent.getComponentType() == org.apache.nifi.registry.flow.ComponentType.CONTROLLER_SERVICE) {
            final String serviceId = ((InstantiatedVersionedControllerService) localComponent).getInstanceId();
            final ControllerServiceNode serviceNode = controllerServiceDAO.getControllerService(serviceId);
            final List<ControllerServiceNode> referencingServices = serviceNode.getReferences().findRecursiveReferences(ControllerServiceNode.class);
            for (final ControllerServiceNode referencingService : referencingServices) {
                affectedComponents.add(createAffectedComponentEntity(referencingService, user));
            }
            final List<ProcessorNode> referencingProcessors = serviceNode.getReferences().findRecursiveReferences(ProcessorNode.class);
            for (final ProcessorNode referencingProcessor : referencingProcessors) {
                affectedComponents.add(createAffectedComponentEntity(referencingProcessor, user));
            }
        }
    }
    // Create a map of all connectable components by versioned component ID to the connectable component itself
    final Map<String, List<Connectable>> connectablesByVersionId = new HashMap<>();
    mapToConnectableId(group.findAllFunnels(), connectablesByVersionId);
    mapToConnectableId(group.findAllInputPorts(), connectablesByVersionId);
    mapToConnectableId(group.findAllOutputPorts(), connectablesByVersionId);
    mapToConnectableId(group.findAllProcessors(), connectablesByVersionId);
    final List<RemoteGroupPort> remotePorts = new ArrayList<>();
    for (final RemoteProcessGroup rpg : group.findAllRemoteProcessGroups()) {
        remotePorts.addAll(rpg.getInputPorts());
        remotePorts.addAll(rpg.getOutputPorts());
    }
    mapToConnectableId(remotePorts, connectablesByVersionId);
    // and the destination (if it exists in the flow currently).
    for (final FlowDifference difference : comparison.getDifferences()) {
        VersionedComponent component = difference.getComponentA();
        if (component == null) {
            component = difference.getComponentB();
        }
        if (component.getComponentType() != org.apache.nifi.registry.flow.ComponentType.CONNECTION) {
            continue;
        }
        final VersionedConnection connection = (VersionedConnection) component;
        final String sourceVersionedId = connection.getSource().getId();
        final List<Connectable> sources = connectablesByVersionId.get(sourceVersionedId);
        if (sources != null) {
            for (final Connectable source : sources) {
                affectedComponents.add(createAffectedComponentEntity(source, user));
            }
        }
        final String destinationVersionId = connection.getDestination().getId();
        final List<Connectable> destinations = connectablesByVersionId.get(destinationVersionId);
        if (destinations != null) {
            for (final Connectable destination : destinations) {
                affectedComponents.add(createAffectedComponentEntity(destination, user));
            }
        }
    }
    return affectedComponents;
}

Example 17 with Connectable

use of org.apache.nifi.connectable.Connectable in project nifi by apache.

the class TestStandardProcessSession method setup.

@Before
public void setup() throws IOException {
    resourceClaimManager = new StandardResourceClaimManager();
    System.setProperty(NiFiProperties.PROPERTIES_FILE_PATH, TestStandardProcessSession.class.getResource("/conf/nifi.properties").getFile());
    final FlowFileEventRepository flowFileEventRepo = Mockito.mock(FlowFileEventRepository.class);
    final CounterRepository counterRepo = Mockito.mock(CounterRepository.class);
    provenanceRepo = new MockProvenanceRepository();
    final Connection connection = createConnection();
    final List<Connection> connList = new ArrayList<>();
    connList.add(connection);
    final ProcessGroup procGroup = Mockito.mock(ProcessGroup.class);
    when(procGroup.getIdentifier()).thenReturn("proc-group-identifier-1");
    connectable = Mockito.mock(Connectable.class);
    when(connectable.hasIncomingConnection()).thenReturn(true);
    when(connectable.getIncomingConnections()).thenReturn(connList);
    when(connectable.getProcessGroup()).thenReturn(procGroup);
    when(connectable.getIdentifier()).thenReturn("connectable-1");
    when(connectable.getConnectableType()).thenReturn(ConnectableType.INPUT_PORT);
    when(connectable.getComponentType()).thenReturn("Unit Test Component");
    Mockito.doAnswer(new Answer<Set<Connection>>() {

        @Override
        public Set<Connection> answer(final InvocationOnMock invocation) throws Throwable {
            final Object[] arguments = invocation.getArguments();
            final Relationship relationship = (Relationship) arguments[0];
            if (relationship == Relationship.SELF) {
                return Collections.emptySet();
            } else if (relationship == FAKE_RELATIONSHIP || relationship.equals(FAKE_RELATIONSHIP)) {
                return null;
            } else {
                return new HashSet<>(connList);
            }
        }
    }).when(connectable).getConnections(Mockito.any(Relationship.class));
    when(connectable.getConnections()).thenReturn(new HashSet<>(connList));
    contentRepo = new MockContentRepository();
    contentRepo.initialize(new StandardResourceClaimManager());
    flowFileRepo = new MockFlowFileRepository();
    context = new RepositoryContext(connectable, new AtomicLong(0L), contentRepo, flowFileRepo, flowFileEventRepo, counterRepo, provenanceRepo);
    session = new StandardProcessSession(context, () -> false);
}

Example 18 with Connectable

use of org.apache.nifi.connectable.Connectable in project nifi by apache.

the class StandardProcessGroup method remove.

@Override
public void remove(final Snippet snippet) {
    writeLock.lock();
    try {
        // ensure that all components are valid
        verifyContents(snippet);
        final Set<Connectable> connectables = getAllConnectables(snippet);
        final Set<String> connectionIdsToRemove = new HashSet<>(getKeys(snippet.getConnections()));
        // Remove all connections that are the output of any Connectable.
        for (final Connectable connectable : connectables) {
            for (final Connection conn : connectable.getConnections()) {
                if (!connections.containsKey(conn.getIdentifier())) {
                    throw new IllegalStateException("Connectable component " + connectable.getIdentifier() + " cannot be removed because it has incoming connections from the parent Process Group");
                }
                connectionIdsToRemove.add(conn.getIdentifier());
            }
        }
        // verify that all connections can be removed
        for (final String id : connectionIdsToRemove) {
            connections.get(id).verifyCanDelete();
        }
        // verify that all processors are stopped and have no active threads
        for (final String procId : snippet.getProcessors().keySet()) {
            final ProcessorNode procNode = getProcessor(procId);
            if (procNode.isRunning()) {
                throw new IllegalStateException("Processor " + procNode.getIdentifier() + " cannot be removed because it is running");
            }
            final int activeThreadCount = scheduler.getActiveThreadCount(procNode);
            if (activeThreadCount != 0) {
                throw new IllegalStateException("Processor " + procNode.getIdentifier() + " cannot be removed because it still has " + activeThreadCount + " active threads");
            }
        }
        // verify that none of the connectables have incoming connections that are not in the Snippet.
        final Set<String> connectionIds = snippet.getConnections().keySet();
        for (final Connectable connectable : connectables) {
            for (final Connection conn : connectable.getIncomingConnections()) {
                if (!connectionIds.contains(conn.getIdentifier()) && !connectables.contains(conn.getSource())) {
                    throw new IllegalStateException("Connectable component " + connectable.getIdentifier() + " cannot be removed because it has incoming connections " + "that are not selected to be deleted");
                }
            }
        }
        // verify that all of the ProcessGroups in the snippet are empty
        for (final String groupId : snippet.getProcessGroups().keySet()) {
            final ProcessGroup toRemove = getProcessGroup(groupId);
            toRemove.verifyCanDelete(true);
        }
        onComponentModified();
        for (final String id : connectionIdsToRemove) {
            removeConnection(connections.get(id));
        }
        for (final String id : getKeys(snippet.getInputPorts())) {
            removeInputPort(inputPorts.get(id));
        }
        for (final String id : getKeys(snippet.getOutputPorts())) {
            removeOutputPort(outputPorts.get(id));
        }
        for (final String id : getKeys(snippet.getFunnels())) {
            removeFunnel(funnels.get(id));
        }
        for (final String id : getKeys(snippet.getLabels())) {
            removeLabel(labels.get(id));
        }
        for (final String id : getKeys(snippet.getProcessors())) {
            removeProcessor(processors.get(id));
        }
        for (final String id : getKeys(snippet.getRemoteProcessGroups())) {
            removeRemoteProcessGroup(remoteGroups.get(id));
        }
        for (final String id : getKeys(snippet.getProcessGroups())) {
            removeProcessGroup(processGroups.get(id));
        }
    } finally {
        writeLock.unlock();
    }
}

Example 19 with Connectable

use of org.apache.nifi.connectable.Connectable in project nifi by apache.

the class StandardProcessGroup method removeConnection.

@Override
public void removeConnection(final Connection connectionToRemove) {
    writeLock.lock();
    try {
        // verify that Connection belongs to this group
        final Connection connection = connections.get(requireNonNull(connectionToRemove).getIdentifier());
        if (connection == null) {
            throw new IllegalStateException("Connection " + connectionToRemove.getIdentifier() + " is not a member of this Process Group");
        }
        connectionToRemove.verifyCanDelete();
        final Connectable source = connectionToRemove.getSource();
        final Connectable dest = connectionToRemove.getDestination();
        // update the source & destination
        source.removeConnection(connection);
        if (source != dest) {
            dest.removeConnection(connection);
        }
        // remove the connection from our map
        connections.remove(connection.getIdentifier());
        LOG.info("{} removed from flow", connection);
        onComponentModified();
        flowController.onConnectionRemoved(connection);
    } finally {
        writeLock.unlock();
    }
}

Example 20 with Connectable

use of org.apache.nifi.connectable.Connectable in project nifi by apache.

the class StandardProcessGroup method isUpdateable.

private boolean isUpdateable(final Connection connection) {
    final Connectable source = connection.getSource();
    if (source.getConnectableType() != ConnectableType.FUNNEL && source.isRunning()) {
        return false;
    }
    final Connectable destination = connection.getDestination();
    if (destination.getConnectableType() != ConnectableType.FUNNEL && destination.isRunning()) {
        return false;
    }
    return true;
}